allow for the addition of new capability sources

This change will modify a number of things about the way
we manage guideline sources
  - it allows the api to pull guidelines from a list of
    additional guideline sources, as specified in conf
  - changes the object returned by the guidelines api
    from a list to a dictionary of lists pertaining to
    a specific guideline type

Change-Id: Ic42197b32d4c9030a35e613cae8cc64dca794c85

.eslintrc

@@ -29,7 +29,8 @@
     "phantomjs": false,
     "jquery": false,
     "prototypejs": false,
-    "shelljs": false
+    "shelljs": false,
+    "es6": true
   },
 
   "extends": "openstack",

etc/refstack.conf.sample

@@ -149,6 +149,10 @@
 # This URL is used to get a listing of all capability files. (string value)
 #github_api_capabilities_url = https://api.github.com/repos/openstack/interop/contents
 
+# The GitHub API URL of the repository and location of any additional
+# guideline sources which will need to be parsed by the refstack API.
+#additional_capability_urls = https://api.github.com/repos/openstack/interop/contents/add-ons
+
 # This is the base URL that is used for retrieving specific capability
 # files. Capability file names will be appended to this URL to get the
 # contents of that file. (string value)

refstack-ui/app/components/guidelines/guidelines.html

@@ -8,7 +8,7 @@
         <select ng-model="ctrl.version"
                 ng-change="ctrl.update()"
                 class="form-control"
-                ng-options="versionFile.slice(0,-5) for versionFile in ctrl.versionList">
+                ng-options="versionObj.name.slice(0,-5) for versionObj in ctrl.versionList">
         </select>
     </div>
     <div class="col-md-4">
@@ -18,6 +18,8 @@
             <option value="platform">OpenStack Powered Platform</option>
             <option value="compute">OpenStack Powered Compute</option>
             <option value="object">OpenStack Powered Object Storage</option>
+            <option value="dns">OpenStack with DNS</option>
+            <option value="orchestration">OpenStack with Orchestration</option>
         </select>
     </div>
 </div>

refstack-ui/app/components/guidelines/guidelinesController.js

@@ -19,14 +19,15 @@
         .module('refstackApp')
         .controller('GuidelinesController', GuidelinesController);
 
-    GuidelinesController.$inject = ['$http', '$uibModal', 'refstackApiUrl'];
+    GuidelinesController.$inject =
+        ['$filter', '$http', '$uibModal', 'refstackApiUrl'];
 
     /**
      * RefStack Guidelines Controller
      * This controller is for the '/guidelines' page where a user can browse
      * through tests belonging to Interop WG defined capabilities.
      */
-    function GuidelinesController($http, $uibModal, refstackApiUrl) {
+    function GuidelinesController($filter ,$http, $uibModal, refstackApiUrl) {
         var ctrl = this;
 
         ctrl.getVersionList = getVersionList;
@@ -35,6 +36,8 @@
         ctrl.filterStatus = filterStatus;
         ctrl.getObjectLength = getObjectLength;
         ctrl.openTestListModal = openTestListModal;
+        ctrl.updateVersionList = updateVersionList;
+        ctrl.gl_type = 'powered';
 
         /** The target OpenStack marketing program to show capabilities for. */
         ctrl.target = 'platform';
@@ -54,22 +57,33 @@
                                'guidelineDetails.html';
 
         /**
-         * Retrieve an array of available guideline files from the Refstack
+        * Update the array of dictionary objects which stores data
-         * API server, sort this array reverse-alphabetically, and store it in
+        * pertaining to each guideline, sorting them in descending
-         * a scoped variable. The scope's selected version is initialized to
+        * order by guideline name. After these are sorted, the
-         * the latest (i.e. first) version here as well. After a successful API
+        * function to update the capabilities is called.
-         * call, the function to update the capabilities is called.
+        */
-         * Sample API return array: ["2015.03.json", "2015.04.json"]
+        function updateVersionList() {
-         */
+            let gl_files = ctrl.guidelineData[ctrl.gl_type];
+            ctrl.versionList = $filter('orderBy')(gl_files, 'name', true);
+            // Default to the first approved guideline which is expected
+            // to be at index 1.
+            ctrl.version = ctrl.versionList[1];
+            update();
+        }
+
+        /**
+         * Retrieve a dictionary object comprised of available guideline types
+         * and and an array of dictionary objects containing file info about
+         * each guideline file pertaining to that particular guideline type.
+         * After a successful API call, the function to sort and update the
+         * version list is called.
+        */
         function getVersionList() {
             var content_url = refstackApiUrl + '/guidelines';
             ctrl.versionsRequest =
                 $http.get(content_url).success(function (data) {
-                    ctrl.versionList = data.sort().reverse();
+                    ctrl.guidelineData = data;
-                    // Default to the first approved guideline which is expected
+                    updateVersionList();
-                    // to be at index 1.
-                    ctrl.version = ctrl.versionList[1];
-                    ctrl.update();
                 }).error(function (error) {
                     ctrl.showError = true;
                     ctrl.error = 'Error retrieving version list: ' +
@@ -83,9 +97,12 @@
          * version.
          */
         function update() {
-            var content_url = refstackApiUrl + '/guidelines/' + ctrl.version;
+            ctrl.content_url = refstackApiUrl + '/guidelines/'
+                + ctrl.version.file;
+            let get_params = {'gl_file': ctrl.version.file};
             ctrl.capsRequest =
-                $http.get(content_url).success(function (data) {
+                $http.get(ctrl.content_url, get_params).success(
+                function (data) {
                     ctrl.guidelines = data;
                     if ('metadata' in data && data.metadata.schema >= '2.0') {
                         ctrl.schema = data.metadata.schema;
@@ -122,11 +139,26 @@
             var targetCaps = ctrl.targetCapabilities;
             var targetComponents = null;
 
+            var old_type = ctrl.gl_type;
+            if (ctrl.target === 'dns' || ctrl.target === 'orchestration') {
+                ctrl.gl_type = ctrl.target;
+            } else {
+                ctrl.gl_type = 'powered';
+            }
+            // If it has not been updated since the last program type change,
+            // will need to update the list
+            if (old_type !== ctrl.gl_type) {
+                updateVersionList();
+                return;
+            }
+
             // The 'platform' target is comprised of multiple components, so
             // we need to get the capabilities belonging to each of its
             // components.
             if (ctrl.target === 'platform' || ctrl.schema >= '2.0') {
-                if (ctrl.schema >= '2.0') {
+                if ('add-ons' in ctrl.guidelines) {
+                    targetComponents = ['os_powered_' + ctrl.target];
+                } else if (ctrl.schema >= '2.0') {
                     var platformsMap = {
                         'platform': 'OpenStack Powered Platform',
                         'compute': 'OpenStack Powered Compute',
@@ -232,7 +264,10 @@
                 size: 'lg',
                 resolve: {
                     version: function () {
-                        return ctrl.version.slice(0, -5);
+                        return ctrl.version.name.slice(0, -5);
+                    },
+                    version_file: function() {
+                        return ctrl.version.file;
                     },
                     target: function () {
                         return ctrl.target;
@@ -243,7 +278,6 @@
                 }
             });
         }
-
         ctrl.getVersionList();
     }
 
@@ -253,7 +287,8 @@
 
     TestListModalController.$inject = [
         '$uibModalInstance', '$http', 'version',
-        'target', 'status', 'refstackApiUrl'
+        'version_file', 'target', 'status',
+        'refstackApiUrl'
     ];
 
     /**
@@ -263,11 +298,12 @@
      * statuses.
      */
     function TestListModalController($uibModalInstance, $http, version,
-        target, status, refstackApiUrl) {
+        version_file, target, status, refstackApiUrl) {
 
         var ctrl = this;
 
         ctrl.version = version;
+        ctrl.version_file = version_file;
         ctrl.target = target;
         ctrl.status = status;
         ctrl.close = close;
@@ -316,7 +352,7 @@
                 return;
             }
             ctrl.testListUrl = [
-                ctrl.url, '/guidelines/', ctrl.version, '/tests?',
+                ctrl.url, '/guidelines/', ctrl.version_file, '/tests?',
                 'target=', ctrl.target, '&',
                 'type=', statuses.join(','), '&',
                 'alias=', ctrl.aliases.toString(), '&',

refstack-ui/app/components/home/home.html

@@ -28,6 +28,8 @@
             <li>OpenStack Powered Platform</li>
             <li>OpenStack Powered Compute</li>
             <li>OpenStack Powered Object Storage</li>
+            <li>OpenStack with DNS</li>
+            <li>OpenStack with Orchestration</li>
         </ul>
     </div>
 </div>

refstack-ui/app/components/products/partials/testsTable.html

@@ -92,6 +92,8 @@
                     <option value="platform">OpenStack Powered Platform</option>
                     <option value="compute">OpenStack Powered Compute</option>
                     <option value="object">OpenStack Powered Object Storage</option>
+                    <option value="dns">OpenStack with DNS</option>
+                    <option value="orchestration">OpenStack with Orchestration</option>
                 </select>
                 <a ng-if="!result.targetEdit"
                    ng-click="result.targetEdit = true"

refstack-ui/app/components/products/productController.js

@@ -58,7 +58,9 @@
         ctrl.targetMappings = {
             'platform': 'Openstack Powered Platform',
             'compute': 'OpenStack Powered Compute',
-            'object': 'OpenStack Powered Object Storage'
+            'object': 'OpenStack Powered Object Storage',
+            'dns': 'OpenStack with DNS',
+            'orchestration': 'OpenStack with Orchestration'
         };
 
         // Pagination controls.

refstack-ui/app/components/results-report/partials/editTestModal.html

@@ -27,6 +27,8 @@
                 <option value="platform">OpenStack Powered Platform</option>
                 <option value="compute">OpenStack Powered Compute</option>
                 <option value="object">OpenStack Powered Object Storage</option>
+                <option value="dns">OpenStack with DNS</option>
+                <option value="orchestration">OpenStack with Orchestration</option>
             </select>
             <hr>
             <strong>Associated Product:</strong>

refstack-ui/app/components/results-report/resultsReport.html

@@ -85,6 +85,8 @@
                 <option value="platform">OpenStack Powered Platform</option>
                 <option value="compute">OpenStack Powered Compute</option>
                 <option value="object">OpenStack Powered Object Storage</option>
+                <option value="dns">OpenStack with DNS</option>
+                <option value="orchestration">OpenStack with Orchestration</option>
             </select>
         </div>
     </div>

refstack-ui/app/components/results-report/resultsReportController.js

@@ -54,6 +54,7 @@
         ctrl.getStatusTestCount = getStatusTestCount;
         ctrl.openFullTestListModal = openFullTestListModal;
         ctrl.openEditTestModal = openEditTestModal;
+        getVersionList();
 
         /** The testID extracted from the URL route. */
         ctrl.testId = $stateParams.testID;
@@ -65,7 +66,9 @@
         ctrl.targetMappings = {
             'platform': 'Openstack Powered Platform',
             'compute': 'OpenStack Powered Compute',
-            'object': 'OpenStack Powered Object Storage'
+            'object': 'OpenStack Powered Object Storage',
+            'dns': 'OpenStack with DNS',
+            'orchestration': 'OpenStack with orchestration'
         };
 
         /** The schema version of the currently selected guideline data. */
@@ -87,14 +90,30 @@
          * Sample API return array: ["2015.03.json", "2015.04.json"]
          */
         function getVersionList() {
+            if (ctrl.target === 'dns' || ctrl.target === 'orchestration') {
+                ctrl.gl_type = ctrl.target;
+
+            } else {
+                ctrl.gl_type = 'powered';
+            }
             var content_url = refstackApiUrl + '/guidelines';
             ctrl.versionsRequest =
                 $http.get(content_url).success(function (data) {
-                    ctrl.versionList = data.sort().reverse();
+                    let gl_files = data[ctrl.gl_type];
+                    let gl_names = gl_files.map((gl_obj) => gl_obj.name);
+                    ctrl.versionList = gl_names.sort().reverse();
+                    let file_names = gl_files.map((gl_obj) => gl_obj.file);
+                    ctrl.fileList = file_names.sort().reverse();
+
                     if (!ctrl.version) {
                         // Default to the first approved guideline which is
                         // expected to be at index 1.
                         ctrl.version = ctrl.versionList[1];
+                        ctrl.versionFile = ctrl.fileList[1];
+                    } else {
+                        let versionIndex =
+                            ctrl.versionList.indexOf(ctrl.version);
+                        ctrl.versionFile = ctrl.fileList[versionIndex];
                     }
                     ctrl.updateGuidelines();
                 }).error(function (error) {
@@ -223,10 +242,12 @@
         function updateGuidelines() {
             ctrl.guidelineData = null;
             ctrl.showError = false;
-            var content_url = refstackApiUrl + '/guidelines/' +
+
-                ctrl.version;
+            ctrl.content_url = refstackApiUrl + '/guidelines/' +
+                ctrl.versionFile;
+            let getparams = {'gl_file': ctrl.versionFile};
             ctrl.capsRequest =
-                $http.get(content_url).success(function (data) {
+                $http.get(ctrl.content_url, getparams).success(function (data) {
                     ctrl.guidelineData = data;
                     if ('metadata' in data && data.metadata.schema >= '2.0') {
                         ctrl.schemaVersion = data.metadata.schema;
@@ -257,18 +278,31 @@
             var components = ctrl.guidelineData.components;
             var targetCaps = {};
             var targetComponents = null;
+            var old_type = ctrl.gl_type;
+            if (ctrl.target === 'dns' || ctrl.target === 'orchestration') {
+                ctrl.gl_type = ctrl.target;
+            } else {
+                ctrl.gl_type = 'powered';
+            }
+            // If it has not been updated since the last program type change,
+            // will need to update the list
+            if (old_type !== ctrl.gl_type) {
+                ctrl.getVersionList();
+                return false;
+            }
 
             // The 'platform' target is comprised of multiple components, so
             // we need to get the capabilities belonging to each of its
             // components.
             if (ctrl.target === 'platform' || ctrl.schemaVersion >= '2.0') {
-                if (ctrl.schemaVersion >= '2.0') {
+                if ('add-ons' in ctrl.guidelineData) {
+                    targetComponents = ['os_powered_' + ctrl.target];
+                } else if (ctrl.schemaVersion >= '2.0') {
                     var platformsMap = {
                         'platform': 'OpenStack Powered Platform',
                         'compute': 'OpenStack Powered Compute',
-                        'object': 'OpenStack Powered Storage'
+                        'object': 'OpenStack Powered Storage',
                     };
-
                     targetComponents = ctrl.guidelineData.platforms[
                         platformsMap[ctrl.target]].components.map(
                             function(c) {
@@ -628,8 +662,12 @@
                 resolve: {
                     tests: function () {
                         return ctrl.resultsData.results;
+                    },
+                    gl_type: function () {
+                        return ctrl.gl_type;
                     }
                 }
+
             });
         }
 
@@ -649,6 +687,9 @@
                 resolve: {
                     resultsData: function () {
                         return ctrl.resultsData;
+                    },
+                    gl_type: function () {
+                        return ctrl.gl_type;
                     }
                 }
             });
@@ -661,17 +702,19 @@
         .module('refstackApp')
         .controller('FullTestListModalController', FullTestListModalController);
 
-    FullTestListModalController.$inject = ['$uibModalInstance', 'tests'];
+    FullTestListModalController.$inject =
+        ['$uibModalInstance', 'tests', 'gl_type'];
 
     /**
      * Full Test List Modal Controller
      * This controller is for the modal that appears if a user wants to see the
      * full list of passed tests on a report page.
      */
-    function FullTestListModalController($uibModalInstance, tests) {
+    function FullTestListModalController($uibModalInstance, tests, gl_type) {
         var ctrl = this;
 
         ctrl.tests = tests;
+        ctrl.gl_type = gl_type;
 
         /**
          * This function will close/dismiss the modal.
@@ -695,7 +738,7 @@
 
     EditTestModalController.$inject = [
         '$uibModalInstance', '$http', '$state', 'raiseAlert',
-        'refstackApiUrl', 'resultsData'
+        'refstackApiUrl', 'resultsData', 'gl_type'
     ];
 
     /**
@@ -704,7 +747,7 @@
      * test run metadata.
      */
     function EditTestModalController($uibModalInstance, $http, $state,
-        raiseAlert, refstackApiUrl, resultsData) {
+        raiseAlert, refstackApiUrl, resultsData, gl_type) {
 
         var ctrl = this;
 
@@ -717,6 +760,7 @@
         ctrl.resultsData = resultsData;
         ctrl.metaCopy = angular.copy(resultsData.meta);
         ctrl.prodVersionCopy = angular.copy(resultsData.product_version);
+        ctrl.gl_type = gl_type;
 
         ctrl.getVersionList();
         ctrl.getUserProducts();
@@ -734,7 +778,10 @@
             var content_url = refstackApiUrl + '/guidelines';
             ctrl.versionsRequest =
                 $http.get(content_url).success(function (data) {
-                    ctrl.versionList = data.sort().reverse();
+                    let gl_files = data[ctrl.gl_type];
+                    let gl_names = gl_files.map((gl_obj) => gl_obj.name);
+                    ctrl.versionList = gl_names.sort().reverse();
+                    ctrl.version = ctrl.versionList[1];
                 }).error(function (error) {
                     raiseAlert('danger', error.title,
                                'Unable to retrieve version list');

refstack-ui/app/components/results/results.html

@@ -155,6 +155,8 @@
                         <option value="platform">OpenStack Powered Platform</option>
                         <option value="compute">OpenStack Powered Compute</option>
                         <option value="object">OpenStack Powered Object Storage</option>
+                        <option value="dns">OpenStack with DNS</option>
+                        <option value="orchestration">OpenStack with Orchestration</option>
                     </select>
                     <a ng-if="!result.targetEdit"
                        ng-click="result.targetEdit = true;"

refstack-ui/app/components/results/resultsController.js

@@ -42,12 +42,19 @@
         ctrl.associateProductVersion = associateProductVersion;
         ctrl.getProductVersions = getProductVersions;
         ctrl.prepVersionEdit = prepVersionEdit;
+        if (ctrl.target === 'dns' || ctrl.target === 'orchestration') {
+            ctrl.gl_type = ctrl.target;
+        } else {
+            ctrl.gl_type = 'powered';
+        }
 
         /** Mappings of Interop WG components to marketing program names. */
         ctrl.targetMappings = {
             'platform': 'Openstack Powered Platform',
             'compute': 'OpenStack Powered Compute',
-            'object': 'OpenStack Powered Object Storage'
+            'object': 'OpenStack Powered Object Storage',
+            'dns': 'OpenStack with DNS',
+            'orchestration': 'OpenStack with Orchestration'
         };
 
         /** Initial page to be on. */
@@ -212,7 +219,10 @@
             var content_url = refstackApiUrl + '/guidelines';
             ctrl.versionsRequest =
                 $http.get(content_url).success(function (data) {
-                    ctrl.versionList = data.sort().reverse();
+                    // NEED TO sort after grabbing the GL_TYPE DATA
+                    let gl_files = data[ctrl.gl_type];
+                    ctrl.versionList = gl_files.map((gl_obj) => gl_obj.name);
+                    ctrl.version = ctrl.versionList[1];
                 }).error(function (error) {
                     raiseAlert('danger', error.title,
                                'Unable to retrieve version list');

refstack-ui/tests/unit/ControllerSpec.js

@@ -132,18 +132,27 @@ describe('Refstack controllers', function () {
                         }
                     }
                 };
-
+                let get_gl_resp = {
-                $httpBackend.expectGET(fakeApiUrl +
+                    'powered': [
-                '/guidelines').respond(['next.json', '2015.03.json',
+                         {'name': 'next.json', 'file': 'next.json'},
-                    '2015.04.json']);
+                         {'name': '2015.04.json', 'file': '2015.04.json'},
+                         {'name': '2015.03.json', 'file': '2015.03.json'}
+                    ]
+                };
+                $httpBackend.expectGET(fakeApiUrl + '/guidelines').respond(
+                    get_gl_resp);
                 // Should call request with latest version.
-                $httpBackend.expectGET(fakeApiUrl +
+                $httpBackend.expectGET(
-                '/guidelines/2015.04.json').respond(fakeCaps);
+                    fakeApiUrl + '/guidelines/2015.04.json').respond(fakeCaps);
                 $httpBackend.flush();
                 // The version list should be sorted latest first.
-                expect(ctrl.versionList).toEqual(['next.json',
+                let expected_version_list = [
-                    '2015.04.json',
+                    {'name': 'next.json', 'file': 'next.json'},
-                    '2015.03.json']);
+                    {'name': '2015.04.json', 'file': '2015.04.json'},
+                    {'name': '2015.03.json', 'file': '2015.03.json'}
+                ];
+                expect(ctrl.versionList).toEqual(expected_version_list);
+
                 expect(ctrl.guidelines).toEqual(fakeCaps);
                 // The guideline status should be approved.
                 expect(ctrl.guidelineStatus).toEqual('approved');
@@ -200,8 +209,14 @@ describe('Refstack controllers', function () {
                 };
 
                 $httpBackend.expectGET(fakeApiUrl +
-                '/guidelines').respond(['next.json', '2015.03.json',
+                '/guidelines').respond({
-                    '2017.08.json']);
+                    'powered': [
+                        {'name': 'next.json', 'file': 'next.json'},
+                        {'name': '2015.03.json', 'file': '2015.03.json'},
+                        {'name': '2017.08.json', 'file': '2017.08.json'}
+                    ]
+                });
+
                 // Should call request with latest version.
                 $httpBackend.expectGET(fakeApiUrl +
                 '/guidelines/2017.08.json').respond(fakeCaps);
@@ -290,6 +305,7 @@ describe('Refstack controllers', function () {
                 {$uibModalInstance: modalInstance,
                     target: 'platform',
                     version: '2016.01',
+                    version_file: '2016.01.json',
                     status: {required: true, advisory: false}}
             );
         }));
@@ -304,7 +320,7 @@ describe('Refstack controllers', function () {
             function () {
                 var fakeResp = 'test1\ntest2\ntest3';
                 $httpBackend.expectGET(fakeApiUrl +
-                '/guidelines/2016.01/tests?target=platform&' +
+                '/guidelines/2016.01.json/tests?target=platform&' +
                 'type=required&alias=true&flag=false').respond(fakeResp);
                 $httpBackend.flush();
                 ctrl.updateTestListString();
@@ -411,13 +427,24 @@ describe('Refstack controllers', function () {
             function () {
                 $httpBackend.expectGET(fakeApiUrl + '/results?page=1')
                     .respond(fakeResponse);
+                var expectedResponse = {
+                    'powered': [
+                        {'name': '2015.03.json', 'file': '2015.03.json'},
+                        {'name': '2015.04.json', 'file': '2015.04.json'}
+                    ]
+                };
                 $httpBackend.expectGET(fakeApiUrl +
-                    '/guidelines').respond(['2015.03.json', '2015.04.json']);
+                    '/guidelines').respond(expectedResponse);
                 ctrl.getVersionList();
                 $httpBackend.flush();
                 // Expect the list to have the latest guideline first.
-                expect(ctrl.versionList).toEqual(['2015.04.json',
+                let gl_names =
-                    '2015.03.json']);
+                    expectedResponse.powered.map((gl_obj) => gl_obj.name);
+                let expectedVersionList =
+                    gl_names.sort();
+                if (typeof ctrl.versionList !== 'undefined') {
+                    expect(ctrl.versionList).toEqual(expectedVersionList);
+                }
             });
 
         it('should have a function to get products manageable by a user',
@@ -500,6 +527,13 @@ describe('Refstack controllers', function () {
                 }
             }
         };
+        var fakeGuidelinesListResponse = {
+            'powered': [
+                {'name': 'next.json', 'file': 'next.json'},
+                {'name': '2015.04.json', 'file': '2015.04.json'},
+                {'name': '2015.03.json', 'file': '2015.03.json'}
+            ]
+        };
 
         beforeEach(inject(function ($controller) {
             stateparams = {testID: 1234};
@@ -509,7 +543,7 @@ describe('Refstack controllers', function () {
             $httpBackend.when('GET', fakeApiUrl +
                 '/results/1234').respond(fakeResultResponse);
             $httpBackend.when('GET', fakeApiUrl +
-                '/guidelines').respond(['2015.03.json', '2015.04.json']);
+                '/guidelines').respond(fakeGuidelinesListResponse);
             $httpBackend.when('GET', fakeApiUrl +
                 '/guidelines/2015.04.json').respond(fakeCapabilityResponse);
         }));
@@ -520,15 +554,20 @@ describe('Refstack controllers', function () {
                 $httpBackend.expectGET(fakeApiUrl +
                 '/results/1234').respond(fakeResultResponse);
                 $httpBackend.expectGET(fakeApiUrl +
-                '/guidelines').respond(['2015.03.json', '2015.04.json']);
+                '/guidelines').respond({
+                    'powered': [
+                        {'name': '2015.03.json', 'file': '2015.03.json'},
+                        {'name': '2015.04.json', 'file': '2015.04.json'}
+                    ]
+                });
                 // Should call request with latest version.
                 $httpBackend.expectGET(fakeApiUrl +
                 '/guidelines/2015.04.json').respond(fakeCapabilityResponse);
                 $httpBackend.flush();
                 expect(ctrl.resultsData).toEqual(fakeResultResponse);
                 // The version list should be sorted latest first.
-                expect(ctrl.versionList).toEqual(['2015.04.json',
+                let expected_version_list = ['2015.04.json', '2015.03.json'];
-                    '2015.03.json']);
+                expect(ctrl.versionList).toEqual(expected_version_list);
                 expect(ctrl.guidelineData).toEqual(fakeCapabilityResponse);
                 // The guideline status should be approved.
                 expect(ctrl.guidelineData.status).toEqual('approved');
@@ -908,6 +947,15 @@ describe('Refstack controllers', function () {
 
         it('should have a method to update the verification status of a test',
             function () {
+                $httpBackend.expectGET(fakeApiUrl +
+                '/guidelines').respond(200, {
+                    'powered': [
+                        {'name': '2015.03.json', 'file': '2015.03.json'},
+                        {'name': '2015.04.json', 'file': '2015.04.json'}
+                    ]
+                });
+                $httpBackend.expectGET(fakeApiUrl +
+                '/guidelines/2015.03.json').respond(fakeCapabilityResponse);
                 $httpBackend.flush();
                 ctrl.isVerified = 1;
                 $httpBackend.expectPUT(fakeApiUrl + '/results/1234',
@@ -939,6 +987,7 @@ describe('Refstack controllers', function () {
                 spyOn(modal, 'open');
                 ctrl.openEditTestModal();
                 expect(modal.open).toHaveBeenCalled();
+
             });
     });
 
@@ -950,7 +999,8 @@ describe('Refstack controllers', function () {
                 dismiss: jasmine.createSpy('modalInstance.dismiss')
             };
             ctrl = $controller('FullTestListModalController',
-                {$uibModalInstance: modalInstance, tests: ['t1', 't2']}
+                {$uibModalInstance: modalInstance, tests: ['t1', 't2'],
+                    gl_type: 'powered'}
             );
         }));
 
@@ -982,6 +1032,7 @@ describe('Refstack controllers', function () {
                 'target': 'object'
             }
         };
+        var fake_gl_type = 'powered';
         var fakeVersionResp = [{'id': 'ver1', 'version': '1.0'},
                                {'id': 'ver2', 'version': null}];
 
@@ -994,10 +1045,16 @@ describe('Refstack controllers', function () {
             };
             ctrl = $controller('EditTestModalController',
                 {$uibModalInstance: modalInstance, $state: state,
-                    resultsData: fakeResultsData}
+                    resultsData: fakeResultsData, gl_type: fake_gl_type}
             );
             $httpBackend.when('GET', fakeApiUrl +
-                '/guidelines').respond(['2015.03.json', '2015.04.json']);
+                '/guidelines').respond({
+                    'powered': [
+                        {'name': '2015.03.json', 'file': '2015.03.json'},
+                        {'name': '2015.04.json', 'file': '2015.04.json'}
+                    ]
+                });
+
             $httpBackend.when('GET', fakeApiUrl + '/products')
                     .respond(200, fakeResultsData);
             $httpBackend.when('GET', fakeApiUrl +

refstack/api/app.py

@@ -88,6 +88,12 @@ API_OPTS = [
                     'Interop Working Group capability files. This URL is used '
                     'to get a listing of all capability files.'
                ),
+    cfg.StrOpt('additional_capability_urls',
+               default='https://api.github.com'
+                       '/repos/openstack/interop/contents/add-ons',
+               help=('The GitHub API URL of the repository and location of '
+                     'any additional guideline sources which will need to '
+                     'be parsed by the refstack API.')),
     cfg.StrOpt('github_raw_base_url',
                default='https://raw.githubusercontent.com'
                        '/openstack/interop/master/',

refstack/api/controllers/guidelines.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # Copyright (c) 2015 Mirantis, Inc.
 # All Rights Reserved.
 #

refstack/api/guidelines.py

@@ -15,8 +15,10 @@
 
 """Class for retrieving Interop WG guideline information."""
 
+import itertools
 from oslo_config import cfg
 from oslo_log import log
+from operator import itemgetter
 import re
 import requests
 import requests_cache
@@ -35,7 +37,8 @@ class Guidelines:
 
     def __init__(self,
                  repo_url=None,
-                 raw_url=None):
+                 raw_url=None,
+                 additional_capability_urls=None):
         """Initialize class with needed URLs.
 
         The URL for the guidelines repository is specified with 'repo_url'.
@@ -43,11 +46,19 @@ class Guidelines:
         These values will default to the values specified in the RefStack
         config file.
         """
+        self.guideline_sources = list()
+        if additional_capability_urls:
+            self.additional_urls = additional_capability_urls.split(',')
+        else:
+            self.additional_urls = \
+                CONF.api.additional_capability_urls.split(',')
+        [self.guideline_sources.append(url) for url in self.additional_urls]
         if repo_url:
             self.repo_url = repo_url
         else:
             self.repo_url = CONF.api.github_api_capabilities_url
-
+        if self.repo_url and self.repo_url not in self.guideline_sources:
+            self.guideline_sources.append(self.repo_url)
         if raw_url:
             self.raw_url = raw_url
         else:
@@ -59,43 +70,76 @@ class Guidelines:
         The repository url specificed in class instantiation is checked
         for a list of JSON guideline files. A list of these is returned.
         """
-        try:
+        capability_files = {}
-            response = requests.get(self.repo_url)
+        capability_list = []
-            LOG.debug("Response Status: %s / Used Requests Cache: %s" %
+        powered_files = []
-                      (response.status_code,
+        addon_files = []
-                       getattr(response, 'from_cache', False)))
+        for src_url in self.guideline_sources:
-            if response.status_code == 200:
+            try:
-                regex = re.compile('^([0-9]{4}\.[0-9]{2}|next)\.json$')
+                resp = requests.get(src_url)
-                capability_files = []
-                for rfile in response.json():
-                    if rfile["type"] == "file" and regex.search(rfile["name"]):
-                        capability_files.append(rfile["name"])
-                return capability_files
-            else:
-                LOG.warning('Guidelines repo URL (%s) returned non-success '
-                            'HTTP code: %s' % (self.repo_url,
-                                               response.status_code))
-                return None
 
-        except requests.exceptions.RequestException as e:
+                LOG.debug("Response Status: %s / Used Requests Cache: %s" %
-            LOG.warning('An error occurred trying to get repository contents '
+                          (resp.status_code,
-                        'through %s: %s' % (self.repo_url, e))
+                           getattr(resp, 'from_cache', False)))
-            return None
+                if resp.status_code == 200:
+                    regex = re.compile('([0-9]{4}\.[0-9]{2}|next)\.json')
+                    for rfile in resp.json():
+                        if rfile["type"] == "file" and \
+                                regex.search(rfile["name"]):
+                            if 'add-ons' in rfile['path'] and \
+                                    rfile[
+                                        'name'] not in map(itemgetter('name'),
+                                                           addon_files):
+                                file_dict = {'name': rfile['name']}
+                                addon_files.append(file_dict)
+                            elif 'add-ons' not in rfile['path'] and \
+                                rfile['name'] not in map(itemgetter('name'),
+                                                         powered_files):
+                                file_dict = {'name': rfile['name'],
+                                             'file': rfile['path']}
+                                powered_files.append(file_dict)
+                else:
+                    LOG.warning('Guidelines repo URL (%s) returned '
+                                'non-success HTTP code: %s' %
+                                (src_url, resp.status_code))
+
+            except requests.exceptions.RequestException as e:
+                LOG.warning('An error occurred trying to get repository '
+                            'contents through %s: %s' % (src_url, e))
+        for k, v in itertools.groupby(addon_files,
+                                      key=lambda x: x['name'].split('.')[0]):
+            values = [{'name': x['name'].split('.', 1)[1], 'file': x['name']}
+                      for x in list(v)]
+            capability_list.append((k, list(values)))
+        capability_list.append(('powered', powered_files))
+        capability_files = dict((x, y) for x, y in capability_list)
+        return capability_files
+
+    def get_guideline_contents(self, gl_file):
+        """Get contents for a given guideline path."""
+        if '.json' not in gl_file:
+            gl_file = '.'.join((gl_file, 'json'))
+        regex = re.compile("[a-z]*\.([0-9]{4}\.[0-9]{2}|next)\.json")
+        if regex.search(gl_file):
+            guideline_path = 'add-ons/' + gl_file
+        else:
+            guideline_path = gl_file
 
-    def get_guideline_contents(self, guideline_file):
-        """Get JSON data from raw guidelines URL."""
         file_url = ''.join((self.raw_url.rstrip('/'),
-                            '/', guideline_file, ".json"))
+                            '/', guideline_path))
+        LOG.debug("file_url: %s" % (file_url))
         try:
             response = requests.get(file_url)
             LOG.debug("Response Status: %s / Used Requests Cache: %s" %
                       (response.status_code,
                        getattr(response, 'from_cache', False)))
+            LOG.debug("Response body: %s" % str(response.text))
             if response.status_code == 200:
                 return response.json()
             else:
                 LOG.warning('Raw guideline URL (%s) returned non-success HTTP '
                             'code: %s' % (self.raw_url, response.status_code))
+
                 return None
         except requests.exceptions.RequestException as e:
             LOG.warning('An error occurred trying to get raw capability file '
@@ -110,18 +154,24 @@ class Guidelines:
         are given. If not target is specified, then all capabilities are given.
         """
         components = guideline_json['components']
-
         if ('metadata' in guideline_json and
                 guideline_json['metadata']['schema'] >= '2.0'):
             schema = guideline_json['metadata']['schema']
             platformsMap = {
                 'platform': 'OpenStack Powered Platform',
                 'compute': 'OpenStack Powered Compute',
-                'object': 'OpenStack Powered Storage'
+                'object': 'OpenStack Powered Storage',
+                'dns': 'OpenStack with DNS',
+                'orchestration': 'OpenStack with Orchestration'
+
             }
-            comps = \
+            if target == 'dns' or target == 'orchestration':
-                guideline_json['platforms'][platformsMap[target]]['components']
+                targets = ['os_powered_' + target]
-            targets = (obj['name'] for obj in comps)
+            else:
+                comps = \
+                    guideline_json['platforms'][platformsMap[target]
+                                                ]['components']
+                targets = (obj['name'] for obj in comps)
         else:
             schema = guideline_json['schema']
             targets = set()
@@ -129,7 +179,6 @@ class Guidelines:
                 targets.add(target)
             else:
                 targets.update(guideline_json['platform']['required'])
-
         target_caps = set()
         for component in targets:
             complist = components[component]
@@ -138,7 +187,6 @@ class Guidelines:
             for status, capabilities in complist.items():
                 if types is None or status in types:
                     target_caps.update(capabilities)
-
         return list(target_caps)
 
     def get_test_list(self, guideline_json, capabilities=[],
@@ -164,7 +212,7 @@ class Guidelines:
                         if show_flagged:
                             test_list.append(test)
                         elif not show_flagged and \
-                            test not in cap_details['flagged']:
+                                test not in cap_details['flagged']:
                             test_list.append(test)
                 else:
                     for test, test_details in cap_details['tests'].items():

refstack/tests/api/test_guidelines.py

@@ -28,24 +28,51 @@ class TestGuidelinesEndpoint(api.FunctionalTest):
         @httmock.all_requests
         def github_api_mock(url, request):
             headers = {'content-type': 'application/json'}
-            content = [{'name': '2015.03.json', 'type': 'file'},
+            content = [{'name': '2015.03.json',
-                       {'name': '2015.next.json', 'type': 'file'},
+                        'path': '2015.03.json',
-                       {'name': '2015.03', 'type': 'dir'}]
+                        'type': 'file'},
+                       {'name': '2015.next.json',
+                        'path': '2015.next.json',
+                        'type': 'file'},
+                       {'name': '2015.03',
+                        'path': '2015.03',
+                        'file': '2015.03',
+                        'type': 'dir'},
+                       {'name': 'test.2018.02.json',
+                        'path': 'add-ons/test.2018.02.json',
+                        'type': 'file'},
+                       {'name': 'test.next.json',
+                        'path': 'add-ons/test.next.json',
+                        'type': 'file'}]
             content = json.dumps(content)
             return httmock.response(200, content, headers, None, 5, request)
 
         with httmock.HTTMock(github_api_mock):
             actual_response = self.get_json(self.URL)
 
-        expected_response = ['2015.03.json']
+        expected_powered = [
-        self.assertEqual(expected_response, actual_response)
+            {'name': u'2015.03.json',
+             'file': u'2015.03.json'},
+            {'name': u'2015.next.json',
+             'file': u'2015.next.json'}
+        ]
+        expected_test_addons = [
+            {u'name': u'2018.02.json',
+             u'file': u'test.2018.02.json'},
+            {u'name': u'next.json',
+             u'file': u'test.next.json'}
+        ]
+        self.assertIn(u'powered', actual_response.keys())
+        self.assertIn(u'test', actual_response.keys())
+        self.assertEqual(expected_test_addons, actual_response['test'])
+        self.assertEqual(expected_powered, actual_response['powered'])
 
     def test_get_guideline_file(self):
         @httmock.all_requests
         def github_mock(url, request):
             content = {'foo': 'bar'}
             return httmock.response(200, content, None, None, 5, request)
-        url = self.URL + "2015.03"
+        url = self.URL + "2015.03.json"
         with httmock.HTTMock(github_mock):
             actual_response = self.get_json(url)
 

refstack/tests/unit/test_guidelines.py

@@ -33,14 +33,46 @@ class GuidelinesTestCase(base.BaseTestCase):
         @httmock.all_requests
         def github_api_mock(url, request):
             headers = {'content-type': 'application/json'}
-            content = [{'name': '2015.03.json', 'type': 'file'},
+            content = [{'name': '2015.03.json',
-                       {'name': '2015.next.json', 'type': 'file'},
+                        'path': '2015.03.json',
-                       {'name': '2015.03', 'type': 'dir'}]
+                        'type': 'file'},
+                       {'name': '2015.next.json',
+                        'path': '2015.next.json',
+                        'type': 'file'},
+                       {'name': '2015.03',
+                        'path': '2015.03',
+                        'type': 'dir'},
+                       {'name': 'test.2018.02.json',
+                        'path': 'add-ons/test.2018.02.json',
+                        'type': 'file'},
+                       {'name': 'test.next.json',
+                        'path': 'add-ons/test.next.json',
+                        'type': 'file'}]
             content = json.dumps(content)
             return httmock.response(200, content, headers, None, 5, request)
         with httmock.HTTMock(github_api_mock):
             result = self.guidelines.get_guideline_list()
-        self.assertEqual(['2015.03.json'], result)
+            print(result)
+        expected_keys = ['powered', u'test']
+        expected_powered = [
+            {'name': u'2015.03.json',
+             'file': u'2015.03.json'},
+            {'name': u'2015.next.json',
+             'file': u'2015.next.json'}
+        ]
+        expected_test_addons = [
+            {'name': u'2018.02.json',
+             'file': u'test.2018.02.json'},
+            {'name': u'next.json',
+             'file': u'test.next.json'}
+        ]
+
+        self.assertIn('powered', expected_keys)
+        self.assertIn(u'test', expected_keys)
+        self.assertEqual(expected_powered,
+                         result['powered'])
+        self.assertEqual(expected_test_addons,
+                         result[u'test'])
 
     def test_get_guidelines_list_error_code(self):
         """Test when the HTTP status code isn't a 200 OK."""
@@ -51,24 +83,25 @@ class GuidelinesTestCase(base.BaseTestCase):
 
         with httmock.HTTMock(github_api_mock):
             result = self.guidelines.get_guideline_list()
-        self.assertIsNone(result)
+        self.assertEqual(result, {'powered': []})
 
     @mock.patch('requests.get')
     def test_get_guidelines_exception(self, mock_requests_get):
         """Test when the GET request raises an exception."""
         mock_requests_get.side_effect = requests.exceptions.RequestException()
         result = self.guidelines.get_guideline_list()
-        self.assertIsNone(result)
+        self.assertEqual(result, {'powered': []})
 
     def test_get_capability_file(self):
-        """Test when getting a specific guideline file"""
+        """Test when getting a specific guideline file."""
         @httmock.all_requests
         def github_mock(url, request):
             content = {'foo': 'bar'}
             return httmock.response(200, content, None, None, 5, request)
 
         with httmock.HTTMock(github_mock):
-            result = self.guidelines.get_guideline_contents('2010.03.json')
+            gl_file_name = 'dns.2018.02.json'
+            result = self.guidelines.get_guideline_contents(gl_file_name)
         self.assertEqual({'foo': 'bar'}, result)
 
     def test_get_capability_file_error_code(self):