Respect history of official projects list
List of official projects constantly changes, if the project became official it appears in the past stats too. To prevent this Stackalytics now checks list of official projects once per release using tags specified in the default_data.json. For initial approach we use election tags for openstack/governance project. Change-Id: I140025775f049fa2be85f42380ab620f84abf267 Partially-Bug: #1497403
This commit is contained in:
Ilya Shakhat
parent
de18e65281
commit
bfee58eb06
@ -15460,51 +15460,123 @@
|
||||
},
|
||||
{
|
||||
"release_name": "Austin",
|
||||
"end_date": "2010-Oct-21"
|
||||
"end_date": "2010-Oct-21",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Bexar",
|
||||
"end_date": "2011-Feb-03"
|
||||
"end_date": "2011-Feb-03",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Cactus",
|
||||
"end_date": "2011-Apr-15"
|
||||
"end_date": "2011-Apr-15",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Diablo",
|
||||
"end_date": "2011-Sep-22"
|
||||
"end_date": "2011-Sep-22",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Essex",
|
||||
"end_date": "2012-Apr-05"
|
||||
"end_date": "2012-Apr-05",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Folsom",
|
||||
"end_date": "2012-Oct-04"
|
||||
"end_date": "2012-Oct-04",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Grizzly",
|
||||
"end_date": "2013-Apr-04"
|
||||
"end_date": "2013-Apr-04",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Havana",
|
||||
"end_date": "2013-Oct-17"
|
||||
"end_date": "2013-Oct-17",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Icehouse",
|
||||
"end_date": "2014-Apr-17"
|
||||
"end_date": "2014-Apr-17",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Juno",
|
||||
"end_date": "2014-Oct-16"
|
||||
"end_date": "2014-Oct-16",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Kilo",
|
||||
"end_date": "2015-Apr-30"
|
||||
"end_date": "2015-Apr-30",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "early_big_tent",
|
||||
"source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Liberty",
|
||||
"end_date": "2015-Oct-15"
|
||||
"end_date": "2015-Oct-15",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "big_tent",
|
||||
"source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"mail_lists": [
|
||||
|
||||
@ -64,6 +64,9 @@
|
||||
},
|
||||
"end_date": {
|
||||
"$ref": "#/definitions/date_format"
|
||||
},
|
||||
"refs": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": ["release_name", "end_date"],
|
||||
|
||||
@ -127,51 +127,123 @@
|
||||
},
|
||||
{
|
||||
"release_name": "Austin",
|
||||
"end_date": "2010-Oct-21"
|
||||
"end_date": "2010-Oct-21",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Bexar",
|
||||
"end_date": "2011-Feb-03"
|
||||
"end_date": "2011-Feb-03",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Cactus",
|
||||
"end_date": "2011-Apr-15"
|
||||
"end_date": "2011-Apr-15",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Diablo",
|
||||
"end_date": "2011-Sep-22"
|
||||
"end_date": "2011-Sep-22",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Essex",
|
||||
"end_date": "2012-Apr-05"
|
||||
"end_date": "2012-Apr-05",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Folsom",
|
||||
"end_date": "2012-Oct-04"
|
||||
"end_date": "2012-Oct-04",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Grizzly",
|
||||
"end_date": "2013-Apr-04"
|
||||
"end_date": "2013-Apr-04",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Havana",
|
||||
"end_date": "2013-Oct-17"
|
||||
"end_date": "2013-Oct-17",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Icehouse",
|
||||
"end_date": "2014-Apr-17"
|
||||
"end_date": "2014-Apr-17",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Juno",
|
||||
"end_date": "2014-Oct-16"
|
||||
"end_date": "2014-Oct-16",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "legacy",
|
||||
"source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Kilo",
|
||||
"end_date": "2015-Apr-30"
|
||||
"end_date": "2015-Apr-30",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "early_big_tent",
|
||||
"source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"release_name": "Liberty",
|
||||
"end_date": "2015-Oct-15"
|
||||
"end_date": "2015-Oct-15",
|
||||
"refs": {
|
||||
"governance": {
|
||||
"type": "big_tent",
|
||||
"source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
|
||||
@ -57,7 +57,7 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
|
||||
</div>
|
||||
|
||||
<div class="drop">
|
||||
<label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
|
||||
<label for="project_type_selector" title="Project type groups modules of same kind: all official (listed in governance's projects.yaml), having the same tag and complementary">Project Type</label>
|
||||
<input type="hidden" id="project_type_selector" style="width: 140px" data-placeholder="Select project type"/>
|
||||
</div>
|
||||
|
||||
|
||||
@ -33,19 +33,51 @@ def _make_module_group(module_groups, name):
|
||||
return m
|
||||
|
||||
|
||||
def read_projects_yaml(project_list_uri):
|
||||
LOG.debug('Process list of projects from uri: %s', project_list_uri)
|
||||
content = yaml.safe_load(utils.read_uri(project_list_uri))
|
||||
module_groups = collections.defaultdict(lambda: {'modules': []})
|
||||
def read_legacy_programs_yaml(module_groups, release_name, content):
|
||||
all_official = module_groups['openstack-official']
|
||||
|
||||
all_official = _make_module_group(module_groups, 'openstack-official')
|
||||
for name, info in six.iteritems(content):
|
||||
# for one program
|
||||
# group_id = name.lower()
|
||||
# if 'codename' in info:
|
||||
# name = '%s (%s)' % (info['codename'], name)
|
||||
# group_id = '%s-group' % info['codename'].lower()
|
||||
#
|
||||
# module_groups[group_id]['module_group_name'] = name
|
||||
# module_groups[group_id]['tag'] = 'program'
|
||||
|
||||
for tag in TAGS:
|
||||
_make_module_group(module_groups, tag)
|
||||
for module in info['projects']:
|
||||
mn = module['repo'].split('/')[1] # module_name
|
||||
|
||||
# module_groups[group_id]['releases'][release_name].append(mn)
|
||||
all_official['releases'][release_name].append(mn)
|
||||
|
||||
|
||||
def read_early_big_tent_projects_yaml(module_groups, release_name, content):
|
||||
all_official = module_groups['openstack-official']
|
||||
|
||||
for name, info in six.iteritems(content):
|
||||
# group_id = '%s-group' % name.lower()
|
||||
# module_groups[group_id]['module_group_name'] = '%s Official' % name
|
||||
# module_groups[group_id]['tag'] = 'program'
|
||||
|
||||
for module in info['projects']:
|
||||
repo_split = module['repo'].split('/')
|
||||
if len(repo_split) < 2:
|
||||
continue # valid repo must be in form of 'org/module'
|
||||
mn = repo_split[1]
|
||||
|
||||
# module_groups[group_id]['releases'][release_name].append(mn)
|
||||
all_official['releases'][release_name].append(mn)
|
||||
|
||||
|
||||
def read_big_tent_projects_yaml(module_groups, release_name, content):
|
||||
all_official = module_groups['openstack-official']
|
||||
|
||||
for name, project in six.iteritems(content):
|
||||
group_id = '%s-group' % name.lower()
|
||||
module_groups[group_id]['module_group_name'] = '%s Official' % name
|
||||
module_groups[group_id]['module_group_name'] = (
|
||||
'%s Official' % name.title())
|
||||
module_groups[group_id]['tag'] = 'program'
|
||||
|
||||
for d_name, deliverable in six.iteritems(project['deliverables']):
|
||||
@ -53,20 +85,66 @@ def read_projects_yaml(project_list_uri):
|
||||
repo_split = repo.split('/')
|
||||
if len(repo_split) < 2:
|
||||
continue # valid repo must be in form of 'org/module'
|
||||
module_name = repo_split[1]
|
||||
|
||||
module_groups[group_id]['modules'].append(module_name)
|
||||
mn = repo_split[1] # module_name
|
||||
|
||||
all_official['modules'].append(module_name)
|
||||
module_groups[group_id]['modules'].append(mn)
|
||||
all_official['releases'][release_name].append(mn)
|
||||
|
||||
tags = deliverable.get('tags', [])
|
||||
for tag in tags:
|
||||
if tag in TAGS:
|
||||
module_groups[tag]['modules'].append(module_name)
|
||||
module_groups[tag]['modules'].append(mn)
|
||||
|
||||
|
||||
def _make_default_module_groups():
|
||||
# create default module groups
|
||||
module_groups = collections.defaultdict(lambda: {'modules': []})
|
||||
|
||||
# openstack official
|
||||
_make_module_group(module_groups, 'openstack-official')
|
||||
module_groups['openstack-official']['releases'] = (
|
||||
collections.defaultdict(list))
|
||||
|
||||
# tags
|
||||
for tag in TAGS:
|
||||
_make_module_group(module_groups, tag)
|
||||
|
||||
return module_groups
|
||||
|
||||
|
||||
GOVERNANCE_PROCESSORS = {
|
||||
'legacy': read_legacy_programs_yaml,
|
||||
'early_big_tent': read_early_big_tent_projects_yaml,
|
||||
'big_tent': read_big_tent_projects_yaml,
|
||||
}
|
||||
|
||||
|
||||
def process_official_list(releases):
|
||||
module_groups = _make_default_module_groups()
|
||||
releases_with_refs = (r for r in releases if r.get('refs'))
|
||||
|
||||
for release in releases_with_refs:
|
||||
ref_governance = release['refs'].get('governance')
|
||||
if not ref_governance:
|
||||
continue
|
||||
|
||||
gov_type = ref_governance['type']
|
||||
gov_source = ref_governance['source']
|
||||
release_name = release['release_name'].lower()
|
||||
|
||||
LOG.debug('Process governance content from uri: %s', gov_source)
|
||||
content = yaml.safe_load(utils.read_uri(gov_source))
|
||||
|
||||
GOVERNANCE_PROCESSORS[gov_type](module_groups, release_name, content)
|
||||
|
||||
# set ids for module groups
|
||||
for group_id, group in six.iteritems(module_groups):
|
||||
group['id'] = group_id
|
||||
group['modules'].sort()
|
||||
|
||||
if 'releases' in group:
|
||||
for gr in six.itervalues(group['releases']):
|
||||
gr.sort()
|
||||
|
||||
return module_groups
|
||||
|
||||
@ -240,8 +240,10 @@ def apply_corrections(uri, runtime_storage_inst):
|
||||
|
||||
def process_project_list(runtime_storage_inst, project_list_uri):
|
||||
module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
|
||||
releases = runtime_storage_inst.get_by_key('releases') or {}
|
||||
|
||||
official_module_groups = governance.process_official_list(releases)
|
||||
|
||||
official_module_groups = governance.read_projects_yaml(project_list_uri)
|
||||
LOG.debug('Update module groups with official: %s', official_module_groups)
|
||||
module_groups.update(official_module_groups)
|
||||
|
||||
|
||||
@ -62,7 +62,7 @@ Sahara:
|
||||
class TestGovernance(testtools.TestCase):
|
||||
|
||||
@mock.patch('stackalytics.processor.utils.read_uri')
|
||||
def test_read_official_projects_yaml(self, read_uri):
|
||||
def test_process_official_list(self, read_uri):
|
||||
read_uri.return_value = SAMPLE
|
||||
|
||||
expected = {
|
||||
@ -95,13 +95,21 @@ class TestGovernance(testtools.TestCase):
|
||||
'openstack-official': {
|
||||
'id': 'openstack-official',
|
||||
'module_group_name': 'openstack-official',
|
||||
'modules': ['python-saharaclient', 'sahara',
|
||||
'sahara-dashboard', 'sahara-extra',
|
||||
'sahara-image-elements', 'sahara-specs'],
|
||||
'modules': [],
|
||||
'releases': {
|
||||
'liberty': ['python-saharaclient', 'sahara',
|
||||
'sahara-dashboard', 'sahara-extra',
|
||||
'sahara-image-elements', 'sahara-specs'],
|
||||
},
|
||||
'tag': 'project_type'
|
||||
},
|
||||
}
|
||||
|
||||
actual = governance.read_projects_yaml('uri')
|
||||
releases = [{
|
||||
'release_name': 'Liberty',
|
||||
'refs': {'governance': {'type': 'big_tent', 'source': 'uri'}}
|
||||
}]
|
||||
|
||||
actual = governance.process_official_list(releases)
|
||||
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user