From da3bada5340bd3f3fdd2538b3ceddef95ac23ae4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 22 Nov 2012 09:38:22 -0800 Subject: [PATCH] Add gerritx project infrastructure. Add all of the support files needed for this to be a project. Also, fix pep8 and pyflakes errors. --- .gitignore | 8 + .gitreview | 4 + .mailmap | 4 + CONTRIBUTING.md | 12 + MANIFEST.in | 8 + README.rst | 7 + gerritx/__init__.py | 0 gerritx/cmd/__init__.py | 13 + gerritx/cmd/close_pull_requests.py | 80 +++--- gerritx/cmd/expire_old_reviews.py | 114 +++++---- gerritx/cmd/fetch_remotes.py | 41 ++-- gerritx/cmd/manage_projects.py | 262 ++++++++++---------- gerritx/cmd/notify_impact.py | 7 +- gerritx/cmd/run_mirror.py | 68 +++--- gerritx/cmd/update_blueprint.py | 54 +++-- gerritx/cmd/update_bug.py | 23 +- gerritx/openstack/__init__.py | 0 gerritx/openstack/common/__init__.py | 0 gerritx/openstack/common/setup.py | 351 +++++++++++++++++++++++++++ gerritx/openstack/common/version.py | 149 ++++++++++++ gerritx/version.py | 20 ++ openstack-common.conf | 7 + setup.cfg | 15 ++ setup.py | 67 +++++ tools/pip-requires | 5 + tools/test-requires | 8 + tox.ini | 24 ++ 27 files changed, 1056 insertions(+), 295 deletions(-) create mode 100644 .gitignore create mode 100644 .gitreview create mode 100644 .mailmap create mode 100644 CONTRIBUTING.md create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 gerritx/__init__.py create mode 100644 gerritx/cmd/__init__.py mode change 100755 => 100644 gerritx/cmd/close_pull_requests.py mode change 100755 => 100644 gerritx/cmd/fetch_remotes.py mode change 100755 => 100644 gerritx/cmd/manage_projects.py mode change 100755 => 100644 gerritx/cmd/notify_impact.py mode change 100755 => 100644 gerritx/cmd/run_mirror.py mode change 100755 => 100644 gerritx/cmd/update_blueprint.py mode change 100755 => 100644 gerritx/cmd/update_bug.py create mode 100644 gerritx/openstack/__init__.py create mode 100644 gerritx/openstack/common/__init__.py create mode 100644 gerritx/openstack/common/setup.py create mode 100644 gerritx/openstack/common/version.py create mode 100644 gerritx/version.py create mode 100644 openstack-common.conf create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tools/pip-requires create mode 100644 tools/test-requires create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e102935 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.tox +build/* +*.pyc +gerritx/versioninfo +AUTHORS +ChangeLog +dist/* +*.egg-info diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..9e9a01d --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack-ci/gerritx.git diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..5606cfb --- /dev/null +++ b/.mailmap @@ -0,0 +1,4 @@ +# Format is: +# +# + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5df2070 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,12 @@ +If you would like to contribute to the development of OpenStack, +you must follow the steps in the "If you're a developer, start here" +section of this page: [http://wiki.openstack.org/HowToContribute](http://wiki.openstack.org/HowToContribute#If_you.27re_a_developer.2C_start_here:) + +Once those steps have been completed, changes to OpenStack +should be submitted for review via the Gerrit tool, following +the workflow documented at [http://wiki.openstack.org/GerritWorkflow](http://wiki.openstack.org/GerritWorkflow). + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed [on Launchpad](https://bugs.launchpad.net/gerritx), +not in GitHub's issue tracker. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..349b889 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include gerritx/versioninfo +include AUTHORS +include ChangeLog + +exclude .gitignore +exclude .gitreview + +global-exclude *.pyc diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..6428582 --- /dev/null +++ b/README.rst @@ -0,0 +1,7 @@ +=============================== +Tools to Manage Gerrit Projects +=============================== + +gerritx is a collection of tools which make managing a gerrit easier. +Specifically, management of gerrit projects and their associated upstream +integration with things like github and launchpad. diff --git a/gerritx/__init__.py b/gerritx/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gerritx/cmd/__init__.py b/gerritx/cmd/__init__.py new file mode 100644 index 0000000..150f9ad --- /dev/null +++ b/gerritx/cmd/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/gerritx/cmd/close_pull_requests.py b/gerritx/cmd/close_pull_requests.py old mode 100755 new mode 100644 index cecbd32..e7742f1 --- a/gerritx/cmd/close_pull_requests.py +++ b/gerritx/cmd/close_pull_requests.py @@ -44,51 +44,59 @@ import os import yaml import logging -logging.basicConfig(level=logging.ERROR) - -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') -GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', - '/etc/github/github.secure.config') - MESSAGE = """Thank you for contributing to OpenStack! %(project)s uses Gerrit for code review. -Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit. +Please visit http://wiki.openstack.org/GerritWorkflow and follow the +instructions there to upload your change to Gerrit. """ -secure_config = ConfigParser.ConfigParser() -secure_config.read(GITHUB_SECURE_CONFIG) -(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] -if secure_config.has_option("github", "oauth_token"): - ghub = github.Github(secure_config.get("github", "oauth_token")) -else: - ghub = github.Github(secure_config.get("github", "username"), - secure_config.get("github", "password")) +def main(): -orgs = ghub.get_user().get_orgs() -orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) -for section in config: - project = section['project'] + logging.basicConfig(level=logging.ERROR) - # Make sure we're supposed to close pull requests for this project: - if 'options' in section and 'has-pull-requests' in section['options']: - continue + PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') + GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', + '/etc/github/github.secure.config') - # Find the project's repo - project_split = project.split('/', 1) - if len(project_split) > 1: - repo = orgs_dict[project_split[0].lower()].get_repo(project_split[1]) + secure_config = ConfigParser.ConfigParser() + secure_config.read(GITHUB_SECURE_CONFIG) + (defaults, config) = [config for config in + yaml.load_all(open(PROJECTS_YAML))] + + if secure_config.has_option("github", "oauth_token"): + ghub = github.Github(secure_config.get("github", "oauth_token")) else: - repo = ghub.get_user().get_repo(project) + ghub = github.Github(secure_config.get("github", "username"), + secure_config.get("github", "password")) - # Close each pull request - pull_requests = repo.get_pulls("open") - for req in pull_requests: - vars = dict(project=project) - issue_data = {"url": repo.url + "/issues/" + str(req.number)} - issue = github.Issue.Issue(req._requester, issue_data, completed = True) - issue.create_comment(MESSAGE % vars) - req.edit(state = "closed") + orgs = ghub.get_user().get_orgs() + orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) + for section in config: + project = section['project'] + + # Make sure we're supposed to close pull requests for this project: + if 'options' in section and 'has-pull-requests' in section['options']: + continue + + # Find the project's repo + project_split = project.split('/', 1) + if len(project_split) > 1: + org = orgs_dict[project_split[0].lower()] + repo = org.get_repo(project_split[1]) + else: + repo = ghub.get_user().get_repo(project) + + # Close each pull request + pull_requests = repo.get_pulls("open") + for req in pull_requests: + vars = dict(project=project) + issue_data = {"url": repo.url + "/issues/" + str(req.number)} + issue = github.Issue.Issue(req._requester, + issue_data, + completed=True) + issue.create_comment(MESSAGE % vars) + req.edit(state="closed") diff --git a/gerritx/cmd/expire_old_reviews.py b/gerritx/cmd/expire_old_reviews.py index f9e908a..b542123 100644 --- a/gerritx/cmd/expire_old_reviews.py +++ b/gerritx/cmd/expire_old_reviews.py @@ -23,56 +23,82 @@ import json import logging import argparse -parser = argparse.ArgumentParser() -parser.add_argument('user', help='The gerrit admin user') -parser.add_argument('ssh_key', help='The gerrit admin SSH key file') -options = parser.parse_args() - -GERRIT_USER = options.user -GERRIT_SSH_KEY = options.ssh_key - -logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/expire_reviews.log') -logger= logging.getLogger('expire_reviews') +logger = logging.getLogger('expire_reviews') logger.setLevel(logging.INFO) -logger.info('Starting expire reviews') -logger.info('Connecting to Gerrit') -ssh = paramiko.SSHClient() -ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) -ssh.connect('localhost', username=GERRIT_USER, key_filename=GERRIT_SSH_KEY, port=29418) +def expire_patch_set(ssh, patch_id, patch_subject, has_negative): + if has_negative: + message = ('code review expired after 1 week of no activity' + ' after a negative review, it can be restored using' + ' the \`Restore Change\` button under the Patch Set' + ' on the web interface') + else: + message = ('code review expired after 2 weeks of no activity,' + ' it can be restored using the \`Restore Change\` button ' + ' under the Patch Set on the web interface') + command = ('gerrit review --abandon' + '--message="{message}" {patch_id}').format( + message=message, + patch_id=patch_id) -def expire_patch_set(patch_id, patch_subject, has_negative): - if has_negative: - message= 'code review expired after 1 week of no activity after a negative review, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface' - else: - message= 'code review expired after 2 weeks of no activity, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface' - command='gerrit review --abandon --message="{0}" {1}'.format(message, patch_id) - logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message) - stdin, stdout, stderr = ssh.exec_command(command) - if stdout.channel.recv_exit_status() != 0: - logger.error(stderr.read()) + logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message) + stdin, stdout, stderr = ssh.exec_command(command) + if stdout.channel.recv_exit_status() != 0: + logger.error(stderr.read()) -# Query all open with no activity for 2 weeks -logger.info('Searching no activity for 2 weeks') -stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --format JSON status:open age:2w') -for line in stdout: - row= json.loads(line) - if not row.has_key('rowCount'): - expire_patch_set(row['currentPatchSet']['revision'], row['subject'], False) +def main(): -# Query all reviewed with no activity for 1 week -logger.info('Searching no activity on negative review for 1 week') -stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --all-approvals --format JSON status:reviewed age:1w') + parser = argparse.ArgumentParser() + parser.add_argument('user', help='The gerrit admin user') + parser.add_argument('ssh_key', help='The gerrit admin SSH key file') + options = parser.parse_args() -for line in stdout: - row= json.loads(line) - if not row.has_key('rowCount'): - # Search for negative approvals - for approval in row['currentPatchSet']['approvals']: - if approval['value'] == '-1': - expire_patch_set(row['currentPatchSet']['revision'], row['subject'], True) - break + GERRIT_USER = options.user + GERRIT_SSH_KEY = options.ssh_key -logger.info('End expire review') + logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s' + ' - %(message)s', + filename='/var/log/gerrit/expire_reviews.log') + + logger.info('Starting expire reviews') + logger.info('Connecting to Gerrit') + + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect('localhost', username=GERRIT_USER, + key_filename=GERRIT_SSH_KEY, port=29418) + + # Query all open with no activity for 2 weeks + logger.info('Searching no activity for 2 weeks') + stdin, stdout, stderr = ssh.exec_command( + 'gerrit query --current-patch-set --format JSON status:open age:2w') + + for line in stdout: + row = json.loads(line) + if 'rowCount' not in row: + expire_patch_set(ssh, + row['currentPatchSet']['revision'], + row['subject'], + False) + + # Query all reviewed with no activity for 1 week + logger.info('Searching no activity on negative review for 1 week') + stdin, stdout, stderr = ssh.exec_command( + 'gerrit query --current-patch-set --all-approvals' + ' --format JSON status:reviewed age:1w') + + for line in stdout: + row = json.loads(line) + if 'rowCount' not in row: + # Search for negative approvals + for approval in row['currentPatchSet']['approvals']: + if approval['value'] == '-1': + expire_patch_set(ssh, + row['currentPatchSet']['revision'], + row['subject'], + True) + break + + logger.info('End expire review') diff --git a/gerritx/cmd/fetch_remotes.py b/gerritx/cmd/fetch_remotes.py old mode 100755 new mode 100644 index 6f8ab8b..f34e518 --- a/gerritx/cmd/fetch_remotes.py +++ b/gerritx/cmd/fetch_remotes.py @@ -33,6 +33,7 @@ import subprocess import shlex import yaml + def run_command(cmd, status=False, env={}): cmd_list = shlex.split(str(cmd)) newenv = os.environ @@ -49,28 +50,30 @@ def run_command_status(cmd, env={}): return run_command(cmd, True, env) -logging.basicConfig(level=logging.ERROR) +def main(): + logging.basicConfig(level=logging.ERROR) -REPO_ROOT = os.environ.get('REPO_ROOT', - '/home/gerrit2/review_site/git') -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') + REPO_ROOT = os.environ.get('REPO_ROOT', + '/home/gerrit2/review_site/git') + PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') -(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] + (defaults, config) = [config for config in + yaml.load_all(open(PROJECTS_YAML))] -for section in config: - project = section['project'] + for section in config: + project = section['project'] - if 'remote' not in section: - continue + if 'remote' not in section: + continue - project_git = "%s.git" % project - os.chdir(os.path.join(REPO_ROOT, project_git)) + project_git = "%s.git" % project + os.chdir(os.path.join(REPO_ROOT, project_git)) - # Make sure that the specified remote exists - remote_url = section['remote'] - # We could check if it exists first, but we're ignoring output anyway - # So just try to make it, and it'll either make a new one or do nothing - run_command("git remote add -f upstream %s" % remote_url) - # Fetch new revs from it - run_command("git remote update upstream") + # Make sure that the specified remote exists + remote_url = section['remote'] + # We could check if it exists first, but we're ignoring output anyway + # So just try to make it, and it'll either make a new one or do nothing + run_command("git remote add -f upstream %s" % remote_url) + # Fetch new revs from it + run_command("git remote update upstream") diff --git a/gerritx/cmd/manage_projects.py b/gerritx/cmd/manage_projects.py old mode 100755 new mode 100644 index 9f5c518..fb1712b --- a/gerritx/cmd/manage_projects.py +++ b/gerritx/cmd/manage_projects.py @@ -50,7 +50,6 @@ import yaml import github import gerritlib.gerrit - logging.basicConfig(level=logging.ERROR) log = logging.getLogger("manage_projects") @@ -113,7 +112,7 @@ def copy_acl_config(project, repo_path, acl_config): status, _ = run_command("cp %s %s" % (acl_config, acl_dest), status=True) if status == 0: - status = git_command(repo_path, "diff --quiet HEAD") + status = git_command(repo_path, "diff-index --quiet HEAD --") if status != 0: return True return False @@ -121,14 +120,14 @@ def copy_acl_config(project, repo_path, acl_config): def push_acl_config(project, remote_url, repo_path, env={}): cmd = "commit -a -m'Update project config.' --author='Openstack Project " \ - "Creator '" + "Creator '" status = git_command(repo_path, cmd) if status != 0: print "Failed to commit config for project: %s" % project return False status, out = git_command_output(repo_path, - "push %s HEAD:refs/meta/config" % - remote_url, env) + "push %s HEAD:refs/meta/config" % + remote_url, env) if status != 0: print "Failed to push config for project: %s" % project print out @@ -194,134 +193,147 @@ def make_ssh_wrapper(gerrit_user, gerrit_key): return dict(GIT_SSH=name) -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') -configs = [config for config in yaml.load_all(open(PROJECTS_YAML))] -defaults = configs[0][0] -default_has_issues = defaults.get('has-issues', False) -default_has_downloads = defaults.get('has-downloads', False) -default_has_wiki = defaults.get('has-wiki', False) +def main(): -LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') -GERRIT_HOST = defaults.get('gerrit-host') -GERRIT_USER = defaults.get('gerrit-user') -GERRIT_KEY = defaults.get('gerrit-key') -GITHUB_SECURE_CONFIG = defaults.get('github-config', - '/etc/github/github-projects.secure.config') + PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') + configs = [config for config in yaml.load_all(open(PROJECTS_YAML))] + defaults = configs[0][0] + default_has_issues = defaults.get('has-issues', False) + default_has_downloads = defaults.get('has-downloads', False) + default_has_wiki = defaults.get('has-wiki', False) -secure_config = ConfigParser.ConfigParser() -secure_config.read(GITHUB_SECURE_CONFIG) + LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') + GERRIT_HOST = defaults.get('gerrit-host') + GERRIT_USER = defaults.get('gerrit-user') + GERRIT_KEY = defaults.get('gerrit-key') -# Project creation doesn't work via oauth -ghub = github.Github(secure_config.get("github", "username"), - secure_config.get("github", "password")) -orgs = ghub.get_user().get_orgs() -orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) + GITHUB_SECURE_CONFIG = defaults.get( + 'github-config', + '/etc/github/github-projects.secure.config') -gerrit = gerritlib.gerrit.Gerrit('localhost', - GERRIT_USER, - 29418, - GERRIT_KEY) -project_list = gerrit.listProjects() -ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) -try: + secure_config = ConfigParser.ConfigParser() + secure_config.read(GITHUB_SECURE_CONFIG) - for section in configs[1]: - project = section['project'] - options = section.get('options', dict()) - description = section.get('description', None) - homepage = section.get('homepage', defaults.get('homepage', None)) - upstream = section.get('upstream', None) + # Project creation doesn't work via oauth + ghub = github.Github(secure_config.get("github", "username"), + secure_config.get("github", "password")) + orgs = ghub.get_user().get_orgs() + orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) - project_git = "%s.git" % project - project_dir = os.path.join(LOCAL_GIT_DIR, project_git) + gerrit = gerritlib.gerrit.Gerrit('localhost', + GERRIT_USER, + 29418, + GERRIT_KEY) + project_list = gerrit.listProjects() + ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) + try: - # Find the project's repo - project_split = project.split('/', 1) - if len(project_split) > 1: - repo_name = project_split[1] - else: - repo_name = project - has_issues = 'has-issues' in options or default_has_issues - has_downloads = 'has-downloads' in options or default_has_downloads - has_wiki = 'has-wiki' in options or default_has_wiki - try: - org = orgs_dict[project_split[0].lower()] - except KeyError: - # We do not have control of this github org ignore the project. - continue - try: - repo = org.get_repo(repo_name) - except github.GithubException: - repo = org.create_repo(repo_name, - homepage=homepage, - has_issues=has_issues, - has_downloads=has_downloads, - has_wiki=has_wiki) - if description: - repo.edit(repo_name, description=description) - if homepage: - repo.edit(repo_name, homepage=homepage) + for section in configs[1]: + project = section['project'] + options = section.get('options', dict()) + description = section.get('description', None) + homepage = section.get('homepage', defaults.get('homepage', None)) + upstream = section.get('upstream', None) - repo.edit(repo_name, has_issues=has_issues, - has_downloads=has_downloads, - has_wiki=has_wiki) + project_git = "%s.git" % project + project_dir = os.path.join(LOCAL_GIT_DIR, project_git) - if 'gerrit' not in [team.name for team in repo.get_teams()]: - teams = org.get_teams() - teams_dict = dict(zip([t.name.lower() for t in teams], teams)) - teams_dict['gerrit'].add_to_repos(repo) - - remote_url = "ssh://localhost:29418/%s" % project - if project not in project_list: - tmpdir = tempfile.mkdtemp() + # Find the project's repo + project_split = project.split('/', 1) + if len(project_split) > 1: + repo_name = project_split[1] + else: + repo_name = project + has_issues = 'has-issues' in options or default_has_issues + has_downloads = 'has-downloads' in options or default_has_downloads + has_wiki = 'has-wiki' in options or default_has_wiki try: - repo_path = os.path.join(tmpdir, 'repo') - if upstream: - run_command("git clone %(upstream)s %(repo_path)s" % - dict(upstream=upstream, repo_path=repo_path)) - else: - run_command("git init %s" % repo_path) - with open(os.path.join(repo_path, - ".gitreview"), 'w') as gitreview: - gitreview.write(""" -[gerrit] -host=%s -port=29418 -project=%s -""" % (GERRIT_HOST, project_git)) - git_command(repo_path, "add .gitreview") - cmd = "commit -a -m'Added .gitreview' --author=" \ - "'Openstack Project Creator " \ - "'" - git_command(repo_path, cmd) - gerrit.createProject(project) - - if not os.path.exists(project_dir): - run_command("git --bare init %s" % project_dir) - run_command("chown -R gerrit2:gerrit2 %s" % project_dir) - - git_command(repo_path, - "push --all %s" % remote_url, - env=ssh_env) - git_command(repo_path, - "push --tags %s" % remote_url, env=ssh_env) - finally: - run_command("rm -fr %s" % tmpdir) - - if 'acl_config' in section: - tmpdir = tempfile.mkdtemp() + org = orgs_dict[project_split[0].lower()] + except KeyError: + # We do not have control of this github org ignore the project. + continue try: - repo_path = os.path.join(tmpdir, 'repo') - ret, _ = run_command_status("git init %s" % repo_path) - if ret != 0: - continue - if (fetch_config(project, remote_url, repo_path, ssh_env) and - copy_acl_config(project, repo_path, - section['acl_config']) and - create_groups_file(project, gerrit, repo_path)): - push_acl_config(project, remote_url, repo_path, ssh_env) - finally: - run_command("rm -fr %s" % tmpdir) -finally: - os.unlink(ssh_env['GIT_SSH']) + repo = org.get_repo(repo_name) + except github.GithubException: + repo = org.create_repo(repo_name, + homepage=homepage, + has_issues=has_issues, + has_downloads=has_downloads, + has_wiki=has_wiki) + if description: + repo.edit(repo_name, description=description) + if homepage: + repo.edit(repo_name, homepage=homepage) + + repo.edit(repo_name, has_issues=has_issues, + has_downloads=has_downloads, + has_wiki=has_wiki) + + if 'gerrit' not in [team.name for team in repo.get_teams()]: + teams = org.get_teams() + teams_dict = dict(zip([t.name.lower() for t in teams], teams)) + teams_dict['gerrit'].add_to_repos(repo) + + remote_url = "ssh://localhost:29418/%s" % project + if project not in project_list: + tmpdir = tempfile.mkdtemp() + try: + repo_path = os.path.join(tmpdir, 'repo') + if upstream: + run_command("git clone %(upstream)s %(repo_path)s" % + dict(upstream=upstream, + repo_path=repo_path)) + else: + run_command("git init %s" % repo_path) + with open(os.path.join(repo_path, + ".gitreview"), + 'w') as gitreview: + gitreview.write(""" + [gerrit] + host=%s + port=29418 + project=%s + """ % (GERRIT_HOST, project_git)) + git_command(repo_path, "add .gitreview") + cmd = "commit -a -m'Added .gitreview' --author=" \ + "'Openstack Project Creator " \ + "'" + git_command(repo_path, cmd) + gerrit.createProject(project) + + if not os.path.exists(project_dir): + run_command("git --bare init %s" % project_dir) + run_command("chown -R gerrit2:gerrit2 %s" + % project_dir) + + git_command(repo_path, + "push --all %s" % remote_url, + env=ssh_env) + git_command(repo_path, + "push --tags %s" % remote_url, env=ssh_env) + finally: + run_command("rm -fr %s" % tmpdir) + + if 'acl_config' in section: + tmpdir = tempfile.mkdtemp() + try: + repo_path = os.path.join(tmpdir, 'repo') + ret, _ = run_command_status("git init %s" % repo_path) + if ret != 0: + continue + if (fetch_config(project, + remote_url, + repo_path, + ssh_env) and + copy_acl_config(project, repo_path, + section['acl_config']) and + create_groups_file(project, gerrit, repo_path)): + push_acl_config(project, + remote_url, + repo_path, + ssh_env) + finally: + run_command("rm -fr %s" % tmpdir) + finally: + os.unlink(ssh_env['GIT_SSH']) diff --git a/gerritx/cmd/notify_impact.py b/gerritx/cmd/notify_impact.py old mode 100755 new mode 100644 index 5c8ba8c..559fd7f --- a/gerritx/cmd/notify_impact.py +++ b/gerritx/cmd/notify_impact.py @@ -34,6 +34,7 @@ Log: %s """ + def process_impact(git_log, args): """Notify mail list of impact""" email_content = EMAIL_TEMPLATE % (args.impact, args.change_url, git_log) @@ -48,10 +49,12 @@ def process_impact(git_log, args): args.dest_address, msg.as_string()) s.quit() + def impacted(git_log, impact_string): """Determine if a changes log indicates there is an impact""" return re.search(impact_string, git_log, re.IGNORECASE) + def extract_git_log(args): """Extract git log of all merged commits""" cmd = ['git', @@ -86,7 +89,3 @@ def main(): # Process impacts found in git log if impacted(git_log, args.impact): process_impact(git_log, args) - - -if __name__ == '__main__': - main() diff --git a/gerritx/cmd/run_mirror.py b/gerritx/cmd/run_mirror.py old mode 100755 new mode 100644 index b6b07e8..d6978de --- a/gerritx/cmd/run_mirror.py +++ b/gerritx/cmd/run_mirror.py @@ -24,6 +24,7 @@ import subprocess import shlex import yaml + def run_command(cmd, status=False, env={}): cmd_list = shlex.split(str(cmd)) newenv = os.environ @@ -40,42 +41,45 @@ def run_command_status(cmd, env={}): return run_command(cmd, True, env) -logging.basicConfig(level=logging.ERROR) +def main(): -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/etc/openstackci/projects.yaml') -PIP_TEMP_DOWNLOAD = os.environ.get('PIP_TEMP_DOWNLOAD', - '/var/lib/pip-download') -GIT_SOURCE = os.environ.get('GIT_SOURCE', 'https://github.com') -pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \ - '--no-install %s' + logging.basicConfig(level=logging.ERROR) -run_command(pip_command % "pip") + PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/etc/openstackci/projects.yaml') + PIP_TEMP_DOWNLOAD = os.environ.get('PIP_TEMP_DOWNLOAD', + '/var/lib/pip-download') + GIT_SOURCE = os.environ.get('GIT_SOURCE', 'https://github.com') + pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \ + '--no-install %s' -(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] + run_command(pip_command % "pip") -for section in config: - project = section['project'] + (defaults, config) = [config for config in + yaml.load_all(open(PROJECTS_YAML))] - os.chdir(PIP_TEMP_DOWNLOAD) - short_project = project.split('/')[1] - if not os.path.isdir(short_project): - run_command("git clone %s/%s.git %s" % (GIT_SOURCE, project, - short_project)) - os.chdir(short_project) - run_command("git fetch origin") + for section in config: + project = section['project'] - for branch in run_command("git branch -a").split("\n"): - branch = branch.strip() - if (not branch.startswith("remotes/origin") - or "origin/HEAD" in branch): - continue - run_command("git reset --hard %s" % branch) - run_command("git clean -x -f -d -q") - print("*********************") - print("Fetching pip requires for %s:%s" % (project, branch)) - for requires_file in ("tools/pip-requires", "tools/test-requires"): - if os.path.exists(requires_file): - stanza = "-r %s" % requires_file - run_command(pip_command % stanza) + os.chdir(PIP_TEMP_DOWNLOAD) + short_project = project.split('/')[1] + if not os.path.isdir(short_project): + run_command("git clone %s/%s.git %s" % (GIT_SOURCE, project, + short_project)) + os.chdir(short_project) + run_command("git fetch origin") + for branch in run_command("git branch -a").split("\n"): + branch = branch.strip() + if (not branch.startswith("remotes/origin") + or "origin/HEAD" in branch): + continue + run_command("git reset --hard %s" % branch) + run_command("git clean -x -f -d -q") + print("*********************") + print("Fetching pip requires for %s:%s" % (project, branch)) + for requires_file in ("tools/pip-requires", + "tools/test-requires"): + if os.path.exists(requires_file): + stanza = "-r %s" % requires_file + run_command(pip_command % stanza) diff --git a/gerritx/cmd/update_blueprint.py b/gerritx/cmd/update_blueprint.py old mode 100755 new mode 100644 index 9c18f1d..64a2e54 --- a/gerritx/cmd/update_blueprint.py +++ b/gerritx/cmd/update_blueprint.py @@ -29,34 +29,39 @@ import ConfigParser import MySQLdb BASE_DIR = '/home/gerrit2/review_site' -GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) +GERRIT_CACHE_DIR = os.path.expanduser( + os.environ.get('GERRIT_CACHE_DIR', + '~/.launchpadlib/cache')) +GERRIT_CREDENTIALS = os.path.expanduser( + os.environ.get('GERRIT_CREDENTIALS', + '~/.launchpadlib/creds')) GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG', - '/home/gerrit2/review_site/etc/gerrit.config') + '/home/gerrit2/review_site/etc/gerrit.config') +GERRIT_SECURE_CONFIG_DEFAULT = '/home/gerrit2/review_site/etc/secure.config' GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG', - '/home/gerrit2/review_site/etc/secure.config') + GERRIT_SECURE_CONFIG_DEFAULT) SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I) BODY_RE = re.compile(r'^\s+.*$') + def get_broken_config(filename): """ gerrit config ini files are broken and have leading tabs """ text = "" - with open(filename,"r") as conf: + with open(filename, "r") as conf: for line in conf.readlines(): text = "%s%s" % (text, line.lstrip()) fp = StringIO.StringIO(text) - c=ConfigParser.ConfigParser() + c = ConfigParser.ConfigParser() c.readfp(fp) return c GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG) SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG) DB_USER = GERRIT_CONFIG.get("database", "username") -DB_PASS = SECURE_CONFIG.get("database","password") -DB_DB = GERRIT_CONFIG.get("database","database") +DB_PASS = SECURE_CONFIG.get("database", "password") +DB_DB = GERRIT_CONFIG.get("database", "database") + def update_spec(launchpad, project, name, subject, link, topic=None): # For testing, if a project doesn't match openstack/foo, use @@ -66,7 +71,8 @@ def update_spec(launchpad, project, name, subject, link, topic=None): project = 'openstack-ci' spec = launchpad.projects[project].getSpecification(name=name) - if not spec: return + if not spec: + return if spec.whiteboard: wb = spec.whiteboard.strip() @@ -74,30 +80,34 @@ def update_spec(launchpad, project, name, subject, link, topic=None): wb = '' changed = False if topic: - topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)], + topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/', 8)], topic) if topiclink not in wb: wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink) changed = True if link not in wb: - wb += "\n\n\nAddressed by: %(link)s\n %(subject)s\n" % dict(subject=subject, - link=link) + wb += ("\n\n\nAddressed by: {link}\n" + " {subject}\n").format(subject=subject, + link=link) changed = True if changed: spec.whiteboard = wb spec.lp_save() + def find_specs(launchpad, dbconn, args): - git_log = subprocess.Popen(['git', - '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', - 'log', '--no-merges', + git_dir_arg = '--git-dir={base_dir}/git/{project}.git'.format( + base_dir=BASE_DIR, + project=args.project) + git_log = subprocess.Popen(['git', git_dir_arg, 'log', '--no-merges', args.commit + '^1..' + args.commit], stdout=subprocess.PIPE).communicate()[0] cur = dbconn.cursor() - cur.execute("select subject, topic from changes where change_key=%s", args.change) + cur.execute("select subject, topic from changes where change_key=%s", + args.change) subject, topic = cur.fetchone() specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)]) @@ -109,6 +119,7 @@ def find_specs(launchpad, dbconn, args): update_spec(launchpad, args.project, spec, subject, args.change_url, topic) + def main(): parser = argparse.ArgumentParser() parser.add_argument('hook') @@ -128,12 +139,9 @@ def main(): launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, GERRIT_CACHE_DIR, - credentials_file = GERRIT_CREDENTIALS, + credentials_file=GERRIT_CREDENTIALS, version='devel') - conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB) + conn = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB) find_specs(launchpad, conn, args) - -if __name__ == '__main__': - main() diff --git a/gerritx/cmd/update_bug.py b/gerritx/cmd/update_bug.py old mode 100755 new mode 100644 index 9479a81..d17bbdd --- a/gerritx/cmd/update_bug.py +++ b/gerritx/cmd/update_bug.py @@ -26,10 +26,12 @@ import subprocess BASE_DIR = '/home/gerrit2/review_site' -GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) +GERRIT_CACHE_DIR = os.path.expanduser( + os.environ.get('GERRIT_CACHE_DIR', + '~/.launchpadlib/cache')) +GERRIT_CREDENTIALS = os.path.expanduser( + os.environ.get('GERRIT_CREDENTIALS', + '~/.launchpadlib/creds')) def add_change_proposed_message(bugtask, change_url, project, branch): @@ -112,7 +114,7 @@ def git2lp(full_project_name): 'openstack-ci/gerrit': 'openstack-ci', 'openstack-ci/lodgeit': 'openstack-ci', 'openstack-ci/meetbot': 'openstack-ci', - } + } return project_map.get(full_project_name, short_project(full_project_name)) @@ -127,7 +129,7 @@ def is_direct_release(full_project_name): 'openstack/openstack-ci-puppet', 'openstack/openstack-manuals', 'openstack/tempest', - ] + ] def process_bugtask(launchpad, bugtask, git_log, args): @@ -147,7 +149,7 @@ def process_bugtask(launchpad, bugtask, git_log, args): # Look for a related task matching the series for reltask in bugtask.related_tasks: if (reltask.bug_target_name.endswith("/" + series) and - reltask.status != u'Fix Released'): + reltask.status != u'Fix Released'): # Use fixcommitted if there is any set_fix_committed(reltask) break @@ -168,7 +170,8 @@ def process_bugtask(launchpad, bugtask, git_log, args): series = args.branch[7:] for reltask in bugtask.related_tasks: if (reltask.bug_target_name.endswith("/" + series) and - reltask.status not in [u'Fix Committed', u'Fix Released']): + reltask.status not in [u'Fix Committed', + u'Fix Released']): set_in_progress(reltask, launchpad, args.uploader, args.change_url) break @@ -238,7 +241,3 @@ def main(): # Process bugtasks found in git log for bugtask in find_bugs(launchpad, git_log, args): process_bugtask(launchpad, bugtask, git_log, args) - - -if __name__ == '__main__': - main() diff --git a/gerritx/openstack/__init__.py b/gerritx/openstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gerritx/openstack/common/__init__.py b/gerritx/openstack/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gerritx/openstack/common/setup.py b/gerritx/openstack/common/setup.py new file mode 100644 index 0000000..2be72e6 --- /dev/null +++ b/gerritx/openstack/common/setup.py @@ -0,0 +1,351 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities with minimum-depends for use in setup.py +""" + +import datetime +import os +import re +import subprocess +import sys + +from setuptools.command import sdist + + +def parse_mailmap(mailmap='.mailmap'): + mapping = {} + if os.path.exists(mailmap): + fp = open(mailmap, 'r') + for l in fp: + l = l.strip() + if not l.startswith('#') and ' ' in l: + canonical_email, alias = [x for x in l.split(' ') + if x.startswith('<')] + mapping[alias] = canonical_email + return mapping + + +def canonicalize_emails(changelog, mapping): + """Takes in a string and an email alias mapping and replaces all + instances of the aliases in the string with their real email. + """ + for alias, email in mapping.iteritems(): + changelog = changelog.replace(alias, email) + return changelog + + +# Get requirements from the first file that exists +def get_reqs_from_files(requirements_files): + reqs_in = [] + for requirements_file in requirements_files: + if os.path.exists(requirements_file): + return open(requirements_file, 'r').read().split('\n') + return [] + + +def parse_requirements(requirements_files=['requirements.txt', + 'tools/pip-requires']): + requirements = [] + for line in get_reqs_from_files(requirements_files): + # For the requirements list, we need to inject only the portion + # after egg= so that distutils knows the package it's looking for + # such as: + # -e git://github.com/openstack/nova/master#egg=nova + if re.match(r'\s*-e\s+', line): + requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + line)) + # such as: + # http://github.com/openstack/nova/zipball/master#egg=nova + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here + elif re.match(r'\s*-f\s+', line): + pass + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass + else: + requirements.append(line) + + return requirements + + +def parse_dependency_links(requirements_files=['requirements.txt', + 'tools/pip-requires']): + dependency_links = [] + # dependency_links inject alternate locations to find packages listed + # in requirements + for line in get_reqs_from_files(requirements_files): + # skip comments and blank lines + if re.match(r'(\s*#)|(\s*$)', line): + continue + # lines with -e or -f need the whole line, minus the flag + if re.match(r'\s*-[ef]\s+', line): + dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) + # lines that are only urls can go in unmolested + elif re.match(r'\s*https?:', line): + dependency_links.append(line) + return dependency_links + + +def write_requirements(): + venv = os.environ.get('VIRTUAL_ENV', None) + if venv is not None: + with open("requirements.txt", "w") as req_file: + output = subprocess.Popen(["pip", "-E", venv, "freeze", "-l"], + stdout=subprocess.PIPE) + requirements = output.communicate()[0].strip() + req_file.write(requirements) + + +def _run_shell_command(cmd): + output = subprocess.Popen(["/bin/sh", "-c", cmd], + stdout=subprocess.PIPE) + out = output.communicate() + if len(out) == 0: + return None + if len(out[0].strip()) == 0: + return None + return out[0].strip() + + +def _get_git_next_version_suffix(branch_name): + datestamp = datetime.datetime.now().strftime('%Y%m%d') + if branch_name == 'milestone-proposed': + revno_prefix = "r" + else: + revno_prefix = "" + _run_shell_command("git fetch origin +refs/meta/*:refs/remotes/meta/*") + milestone_cmd = "git show meta/openstack/release:%s" % branch_name + milestonever = _run_shell_command(milestone_cmd) + if not milestonever: + milestonever = "" + post_version = _get_git_post_version() + # post version should look like: + # 0.1.1.4.gcc9e28a + # where the bit after the last . is the short sha, and the bit between + # the last and second to last is the revno count + (revno, sha) = post_version.split(".")[-2:] + first_half = "%(milestonever)s~%(datestamp)s" % locals() + second_half = "%(revno_prefix)s%(revno)s.%(sha)s" % locals() + return ".".join((first_half, second_half)) + + +def _get_git_current_tag(): + return _run_shell_command("git tag --contains HEAD") + + +def _get_git_tag_info(): + return _run_shell_command("git describe --tags") + + +def _get_git_post_version(): + current_tag = _get_git_current_tag() + if current_tag is not None: + return current_tag + else: + tag_info = _get_git_tag_info() + if tag_info is None: + base_version = "0.0" + cmd = "git --no-pager log --oneline" + out = _run_shell_command(cmd) + revno = len(out.split("\n")) + sha = _run_shell_command("git describe --always") + else: + tag_infos = tag_info.split("-") + base_version = "-".join(tag_infos[:-2]) + (revno, sha) = tag_infos[-2:] + return "%s.%s.%s" % (base_version, revno, sha) + + +def write_git_changelog(): + """Write a changelog based on the git changelog.""" + if os.path.isdir('.git'): + git_log_cmd = 'git log --stat' + changelog = _run_shell_command(git_log_cmd) + mailmap = parse_mailmap() + with open("ChangeLog", "w") as changelog_file: + changelog_file.write(canonicalize_emails(changelog, mailmap)) + + +def generate_authors(): + """Create AUTHORS file using git commits.""" + jenkins_email = 'jenkins@review.openstack.org' + old_authors = 'AUTHORS.in' + new_authors = 'AUTHORS' + if os.path.isdir('.git'): + # don't include jenkins email address in AUTHORS file + git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | " + "grep -v " + jenkins_email) + changelog = _run_shell_command(git_log_cmd) + mailmap = parse_mailmap() + with open(new_authors, 'w') as new_authors_fh: + new_authors_fh.write(canonicalize_emails(changelog, mailmap)) + if os.path.exists(old_authors): + with open(old_authors, "r") as old_authors_fh: + new_authors_fh.write('\n' + old_authors_fh.read()) + +_rst_template = """%(heading)s +%(underline)s + +.. automodule:: %(module)s + :members: + :undoc-members: + :show-inheritance: +""" + + +def read_versioninfo(project): + """Read the versioninfo file. If it doesn't exist, we're in a github + zipball, and there's really no way to know what version we really + are, but that should be ok, because the utility of that should be + just about nil if this code path is in use in the first place.""" + versioninfo_path = os.path.join(project, 'versioninfo') + if os.path.exists(versioninfo_path): + with open(versioninfo_path, 'r') as vinfo: + version = vinfo.read().strip() + else: + version = "0.0.0" + return version + + +def write_versioninfo(project, version): + """Write a simple file containing the version of the package.""" + open(os.path.join(project, 'versioninfo'), 'w').write("%s\n" % version) + + +def get_cmdclass(): + """Return dict of commands to run from setup.py.""" + + cmdclass = dict() + + def _find_modules(arg, dirname, files): + for filename in files: + if filename.endswith('.py') and filename != '__init__.py': + arg["%s.%s" % (dirname.replace('/', '.'), + filename[:-3])] = True + + class LocalSDist(sdist.sdist): + """Builds the ChangeLog and Authors files from VC first.""" + + def run(self): + write_git_changelog() + generate_authors() + # sdist.sdist is an old style class, can't use super() + sdist.sdist.run(self) + + cmdclass['sdist'] = LocalSDist + + # If Sphinx is installed on the box running setup.py, + # enable setup.py to build the documentation, otherwise, + # just ignore it + try: + from sphinx.setup_command import BuildDoc + + class LocalBuildDoc(BuildDoc): + def generate_autoindex(self): + print "**Autodocumenting from %s" % os.path.abspath(os.curdir) + modules = {} + option_dict = self.distribution.get_option_dict('build_sphinx') + source_dir = os.path.join(option_dict['source_dir'][1], 'api') + if not os.path.exists(source_dir): + os.makedirs(source_dir) + for pkg in self.distribution.packages: + if '.' not in pkg: + os.path.walk(pkg, _find_modules, modules) + module_list = modules.keys() + module_list.sort() + autoindex_filename = os.path.join(source_dir, 'autoindex.rst') + with open(autoindex_filename, 'w') as autoindex: + autoindex.write(""".. toctree:: + :maxdepth: 1 + +""") + for module in module_list: + output_filename = os.path.join(source_dir, + "%s.rst" % module) + heading = "The :mod:`%s` Module" % module + underline = "=" * len(heading) + values = dict(module=module, heading=heading, + underline=underline) + + print "Generating %s" % output_filename + with open(output_filename, 'w') as output_file: + output_file.write(_rst_template % values) + autoindex.write(" %s.rst\n" % module) + + def run(self): + if not os.getenv('SPHINX_DEBUG'): + self.generate_autoindex() + + for builder in ['html', 'man']: + self.builder = builder + self.finalize_options() + self.project = self.distribution.get_name() + self.version = self.distribution.get_version() + self.release = self.distribution.get_version() + BuildDoc.run(self) + cmdclass['build_sphinx'] = LocalBuildDoc + except ImportError: + pass + + return cmdclass + + +def get_git_branchname(): + for branch in _run_shell_command("git branch --color=never").split("\n"): + if branch.startswith('*'): + _branch_name = branch.split()[1].strip() + if _branch_name == "(no": + _branch_name = "no-branch" + return _branch_name + + +def get_pre_version(projectname, base_version): + """Return a version which is leading up to a version that will + be released in the future.""" + if os.path.isdir('.git'): + current_tag = _get_git_current_tag() + if current_tag is not None: + version = current_tag + else: + branch_name = os.getenv('BRANCHNAME', + os.getenv('GERRIT_REFNAME', + get_git_branchname())) + version_suffix = _get_git_next_version_suffix(branch_name) + version = "%s~%s" % (base_version, version_suffix) + write_versioninfo(projectname, version) + return version + else: + version = read_versioninfo(projectname) + return version + + +def get_post_version(projectname): + """Return a version which is equal to the tag that's on the current + revision if there is one, or tag plus number of additional revisions + if the current revision has no tag.""" + + if os.path.isdir('.git'): + version = _get_git_post_version() + write_versioninfo(projectname, version) + return version + return read_versioninfo(projectname) diff --git a/gerritx/openstack/common/version.py b/gerritx/openstack/common/version.py new file mode 100644 index 0000000..ca350ed --- /dev/null +++ b/gerritx/openstack/common/version.py @@ -0,0 +1,149 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities for consuming the auto-generated versioninfo files. +""" + +import datetime +import pkg_resources +import os + +import setup + + +class _deferred_version_string(object): + """Internal helper class which provides delayed version calculation.""" + def __init__(self, version_info, prefix): + self.version_info = version_info + self.prefix = prefix + + def __str__(self): + return "%s%s" % (self.prefix, self.version_info.version_string()) + + def __repr__(self): + return "%s%s" % (self.prefix, self.version_info.version_string()) + + +class VersionInfo(object): + + def __init__(self, package, python_package=None, pre_version=None): + """Object that understands versioning for a package + :param package: name of the top level python namespace. For glance, + this would be "glance" for python-glanceclient, it + would be "glanceclient" + :param python_package: optional name of the project name. For + glance this can be left unset. For + python-glanceclient, this would be + "python-glanceclient" + :param pre_version: optional version that the project is working to + """ + self.package = package + if python_package is None: + self.python_package = package + else: + self.python_package = python_package + self.pre_version = pre_version + self.version = None + + def _generate_version(self): + """Defer to the openstack.common.setup routines for making a + version from git.""" + if self.pre_version is None: + return setup.get_post_version(self.python_package) + else: + return setup.get_pre_version(self.python_package, self.pre_version) + + def _newer_version(self, pending_version): + """Check to see if we're working with a stale version or not. + We expect a version string that either looks like: + 2012.2~f3~20120708.10.4426392 + which is an unreleased version of a pre-version, or: + 0.1.1.4.gcc9e28a + which is an unreleased version of a post-version, or: + 0.1.1 + Which is a release and which should match tag. + For now, if we have a date-embedded version, check to see if it's + old, and if so re-generate. Otherwise, just deal with it. + """ + try: + version_date = int(self.version.split("~")[-1].split('.')[0]) + if version_date < int(datetime.date.today().strftime('%Y%m%d')): + return self._generate_version() + else: + return pending_version + except Exception: + return pending_version + + def version_string_with_vcs(self, always=False): + """Return the full version of the package including suffixes indicating + VCS status. + + For instance, if we are working towards the 2012.2 release, + canonical_version_string should return 2012.2 if this is a final + release, or else something like 2012.2~f1~20120705.20 if it's not. + + :param always: if true, skip all version caching + """ + if always: + self.version = self._generate_version() + + if self.version is None: + + requirement = pkg_resources.Requirement.parse(self.python_package) + versioninfo = "%s/versioninfo" % self.package + try: + raw_version = pkg_resources.resource_string(requirement, + versioninfo) + self.version = self._newer_version(raw_version.strip()) + except (IOError, pkg_resources.DistributionNotFound): + self.version = self._generate_version() + + return self.version + + def canonical_version_string(self, always=False): + """Return the simple version of the package excluding any suffixes. + + For instance, if we are working towards the 2012.2 release, + canonical_version_string should return 2012.2 in all cases. + + :param always: if true, skip all version caching + """ + return self.version_string_with_vcs(always).split('~')[0] + + def version_string(self, always=False): + """Return the base version of the package. + + For instance, if we are working towards the 2012.2 release, + version_string should return 2012.2 if this is a final release, or + 2012.2-dev if it is not. + + :param always: if true, skip all version caching + """ + version_parts = self.version_string_with_vcs(always).split('~') + if len(version_parts) == 1: + return version_parts[0] + else: + return '%s-dev' % (version_parts[0],) + + def deferred_version_string(self, prefix=""): + """Generate an object which will expand in a string context to + the results of version_string(). We do this so that don't + call into pkg_resources every time we start up a program when + passing version information into the CONF constructor, but + rather only do the calculation when and if a version is requested + """ + return _deferred_version_string(self, prefix) diff --git a/gerritx/version.py b/gerritx/version.py new file mode 100644 index 0000000..32af0c0 --- /dev/null +++ b/gerritx/version.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gerritx.openstack.common import version as common_version + +version_info = common_version.VersionInfo('gerritx') diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 0000000..8f55e0d --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,7 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +modules=setup,version + +# The base module to hold the copy of openstack.common +base=gerritx diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..93c964c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,15 @@ +[nosetests] +cover-package = gerritx +cover-html = true +cover-erase = true +cover-inclusive = true +verbosity=2 +detailed-errors=1 + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = doc/build/html diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c1a15ce --- /dev/null +++ b/setup.py @@ -0,0 +1,67 @@ +# Copyright (c) 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import setuptools + + +from gerritx.openstack.common import setup +from gerritx.version import version_info as version + +requires = setup.parse_requirements() +tests_require = setup.parse_requirements(['tools/test-requires']) +depend_links = setup.parse_dependency_links() + + +def read_file(file_name): + return open(os.path.join(os.path.dirname(__file__), file_name)).read() + + +setuptools.setup( + name="gerritx", + version=version.canonical_version_string(always=True), + author='Hewlett-Packard Development Company, L.P.', + author_email='openstack@lists.launchpad.net', + description="Tools for managing gerrit projects and external sources.", + license="Apache License, Version 2.0", + url="https://github.com/openstack-ci/gerritx", + packages=setuptools.find_packages(exclude=['tests', 'tests.*']), + include_package_data=True, + setup_requires=['setuptools_git>=0.4'], + cmdclass=setup.get_cmdclass(), + install_requires=requires, + dependency_links=depend_links, + tests_require=tests_require, + test_suite="nose.collector", + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python" + ], + entry_points={ + 'console_scripts': [ + 'close-pull-requests=gerritx.cmd.close_pull_requests:main', + 'expire-old-reviews=gerritx.cmd.expire_old_reviews:main', + 'fetch-remotes=gerritx.cmd.fetch_remotes:main', + 'manage-projects=gerritx.cmd.manage_projects:main', + 'notify-impact=gerritx.cmd.notify_impact:main', + 'run-mirror=gerritx.cmd.run_mirror:main', + 'update-blueprint=gerritx.cmd.update_blueprint:main', + 'update-bug=gerritx.cmd.update_bug:main', + ], + } +) diff --git a/tools/pip-requires b/tools/pip-requires new file mode 100644 index 0000000..e990c7f --- /dev/null +++ b/tools/pip-requires @@ -0,0 +1,5 @@ +gerritlib +MySQL-python +paramiko +PyGithub +pyyaml diff --git a/tools/test-requires b/tools/test-requires new file mode 100644 index 0000000..a2789cd --- /dev/null +++ b/tools/test-requires @@ -0,0 +1,8 @@ +distribute>=0.6.24 + +fixtures +pep8==1.3 +pyflakes +python-subunit +sphinx>=1.1.2 +testtools diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0f59a0f --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +envlist = py26,py27,pep8 + +[testenv] +setenv = VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/tools/pip-requires + -r{toxinidir}/tools/test-requires +commands = nosetests {posargs} + +[testenv:pep8] +deps = pep8==1.3.3 +commands = + pep8 --exclude=.venv,.tox,dist,doc,openstack,*lib/python*,*egg \ + --repeat --show-source . + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 + +[testenv:pyflakes] +deps = pyflakes +commands = pyflakes gerritx/cmd + +[testenv:venv] +commands = {posargs}