diff --git a/MANIFEST.in b/MANIFEST.in index 898ecd7..74fc557 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include jeepyb/versioninfo include AUTHORS include ChangeLog diff --git a/README.rst b/README.rst index ef52cf5..14f9d2a 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,12 @@ -=============================== -Tools to Manage Gerrit Projects -=============================== +==================== +Partial PyPI Mirrors +==================== -jeepyb is a collection of tools which make managing a gerrit easier. -Specifically, management of gerrit projects and their associated upstream -integration with things like github and launchpad. +Sometimes you want a PyPI mirror, but you don't want the whole thing. You +certainly don't want external links. What you want are the things that you +need and nothing more. What's more, you often know exactly what you need +because you already have a pip requirements.txt file containing the list of +things you expect to download from PyPI. + +pypi-mirror will build a local static mirror for you based on requirements +files in git repos. diff --git a/jeepyb/cmd/close_pull_requests.py b/jeepyb/cmd/close_pull_requests.py deleted file mode 100644 index 8abeda9..0000000 --- a/jeepyb/cmd/close_pull_requests.py +++ /dev/null @@ -1,105 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Github pull requests closer reads a project config file called projects.yaml -# It should look like: - -# - homepage: http://openstack.org -# team-id: 153703 -# has-wiki: False -# has-issues: False -# has-downloads: False -# --- -# - project: PROJECT_NAME -# options: -# - has-pull-requests - -# Github authentication information is read from github.secure.config, -# which should look like: - -# [github] -# username = GITHUB_USERNAME -# password = GITHUB_PASSWORD -# -# or -# -# [github] -# oauth_token = GITHUB_OAUTH_TOKEN - -import ConfigParser -import github -import logging -import os -import yaml - -MESSAGE = """Thank you for contributing to %(project)s! - -%(project)s uses Gerrit for code review. - -Please visit http://wiki.openstack.org/GerritWorkflow and follow the -instructions there to upload your change to Gerrit. -""" - - -def main(): - - logging.basicConfig(level=logging.ERROR) - - PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', - '/etc/github/github.secure.config') - - secure_config = ConfigParser.ConfigParser() - secure_config.read(GITHUB_SECURE_CONFIG) - (defaults, config) = [config for config in - yaml.load_all(open(PROJECTS_YAML))] - - if secure_config.has_option("github", "oauth_token"): - ghub = github.Github(secure_config.get("github", "oauth_token")) - else: - ghub = github.Github(secure_config.get("github", "username"), - secure_config.get("github", "password")) - - orgs = ghub.get_user().get_orgs() - orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) - for section in config: - project = section['project'] - - # Make sure we're supposed to close pull requests for this project: - if 'options' in section and 'has-pull-requests' in section['options']: - continue - - # Find the project's repo - project_split = project.split('/', 1) - if len(project_split) > 1: - org = orgs_dict[project_split[0].lower()] - repo = org.get_repo(project_split[1]) - else: - repo = ghub.get_user().get_repo(project) - - # Close each pull request - pull_requests = repo.get_pulls("open") - for req in pull_requests: - vars = dict(project=project) - issue_data = {"url": repo.url + "/issues/" + str(req.number)} - issue = github.Issue.Issue(req._requester, - issue_data, - completed=True) - issue.create_comment(MESSAGE % vars) - req.edit(state="closed") - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/create_cgitrepos.py b/jeepyb/cmd/create_cgitrepos.py deleted file mode 100644 index 3fc4f19..0000000 --- a/jeepyb/cmd/create_cgitrepos.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# create_cgitrepos.py reads the project config file called projects.yaml -# and generates a cgitrepos configuration file which is then copied to -# the cgit server. -# -# It also creates the necessary top-level directories for each project -# organization (openstack, stackforge, etc) - -import os -import subprocess -import yaml - - -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/cgit/projects.yaml') -CGIT_REPOS = os.environ.get('CGIT_REPOS', - '/etc/cgitrepos') -REPO_PATH = os.environ.get('REPO_PATH', - '/var/lib/git') -CGIT_USER = os.environ.get('CGIT_USER', 'cgit') -CGIT_GROUP = os.environ.get('CGIT_GROUP', 'cgit') - - -def main(): - (defaults, config) = tuple(yaml.safe_load_all(open(PROJECTS_YAML))) - gitorgs = {} - names = set() - for entry in config: - (org, name) = entry['project'].split('/') - description = entry.get('description', name) - assert name not in names - names.add(name) - gitorgs.setdefault(org, []).append((name, description)) - for org in gitorgs: - if not os.path.isdir('%s/%s' % (REPO_PATH, org)): - os.makedirs('%s/%s' % (REPO_PATH, org)) - with open(CGIT_REPOS, 'w') as cgit_file: - cgit_file.write('# Autogenerated by create_cgitrepos.py\n') - for org in sorted(gitorgs): - cgit_file.write('\n') - cgit_file.write('section=%s\n' % (org)) - org_dir = os.path.join(REPO_PATH, org) - projects = gitorgs[org] - projects.sort() - for (name, description) in projects: - project_repo = "%s.git" % os.path.join(org_dir, name) - cgit_file.write('\n') - cgit_file.write('repo.url=%s/%s\n' % (org, name)) - cgit_file.write('repo.path=%s/\n' % (project_repo)) - cgit_file.write('repo.desc=%s\n' % (description)) - if not os.path.exists(project_repo): - subprocess.call(['git', 'init', '--bare', project_repo]) - subprocess.call(['chown', '-R', '%s:%s' - % (CGIT_USER, CGIT_GROUP), project_repo]) - - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/expire_old_reviews.py b/jeepyb/cmd/expire_old_reviews.py deleted file mode 100644 index 77f9856..0000000 --- a/jeepyb/cmd/expire_old_reviews.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This script is designed to expire old code reviews that have not been touched -# using the following rules: -# 1. if negative comment and no activity in 1 week, expire - -import argparse -import json -import logging -import paramiko - -logger = logging.getLogger('expire_reviews') -logger.setLevel(logging.INFO) - - -def expire_patch_set(ssh, patch_id, patch_subject): - message = ('code review expired after 1 week of no activity' - ' after a negative review, it can be restored using' - ' the \`Restore Change\` button under the Patch Set' - ' on the web interface') - command = ('gerrit review --abandon ' - '--message="{message}" {patch_id}').format( - message=message, - patch_id=patch_id) - - logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message) - stdin, stdout, stderr = ssh.exec_command(command) - if stdout.channel.recv_exit_status() != 0: - logger.error(stderr.read()) - - -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument('user', help='The gerrit admin user') - parser.add_argument('ssh_key', help='The gerrit admin SSH key file') - options = parser.parse_args() - - GERRIT_USER = options.user - GERRIT_SSH_KEY = options.ssh_key - - logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s' - ' - %(message)s', - filename='/var/log/gerrit/expire_reviews.log') - - logger.info('Starting expire reviews') - logger.info('Connecting to Gerrit') - - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect('localhost', username=GERRIT_USER, - key_filename=GERRIT_SSH_KEY, port=29418) - - # Query all reviewed with no activity for 1 week - logger.info('Searching no activity on negative review for 1 week') - stdin, stdout, stderr = ssh.exec_command( - 'gerrit query --current-patch-set --all-approvals' - ' --format JSON status:reviewed age:1w') - - for line in stdout: - row = json.loads(line) - if 'rowCount' not in row: - # Search for negative approvals - for approval in row['currentPatchSet']['approvals']: - if approval['value'] in ('-1', '-2'): - expire_patch_set(ssh, - row['currentPatchSet']['revision'], - row['subject']) - break - - logger.info('End expire review') - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/fetch_remotes.py b/jeepyb/cmd/fetch_remotes.py deleted file mode 100644 index 7829097..0000000 --- a/jeepyb/cmd/fetch_remotes.py +++ /dev/null @@ -1,82 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Fetch remotes reads a project config file called projects.yaml -# It should look like: - -# - homepage: http://openstack.org -# team-id: 153703 -# has-wiki: False -# has-issues: False -# has-downloads: False -# --- -# - project: PROJECT_NAME -# options: -# - remote: https://gerrit.googlesource.com/gerrit - - -import logging -import os -import shlex -import subprocess -import yaml - - -def run_command(cmd, status=False, env={}): - cmd_list = shlex.split(str(cmd)) - newenv = os.environ - newenv.update(env) - p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def run_command_status(cmd, env={}): - return run_command(cmd, True, env) - - -def main(): - logging.basicConfig(level=logging.ERROR) - - REPO_ROOT = os.environ.get('REPO_ROOT', - '/home/gerrit2/review_site/git') - PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - - (defaults, config) = [config for config in - yaml.load_all(open(PROJECTS_YAML))] - - for section in config: - project = section['project'] - - if 'remote' not in section: - continue - - project_git = "%s.git" % project - os.chdir(os.path.join(REPO_ROOT, project_git)) - - # Make sure that the specified remote exists - remote_url = section['remote'] - # We could check if it exists first, but we're ignoring output anyway - # So just try to make it, and it'll either make a new one or do nothing - run_command("git remote add -f upstream %s" % remote_url) - # Fetch new revs from it - run_command("git remote update upstream") - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py deleted file mode 100644 index a05229c..0000000 --- a/jeepyb/cmd/manage_projects.py +++ /dev/null @@ -1,412 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# Copyright (c) 2012 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# manage_projects.py reads a project config file called projects.yaml -# It should look like: - -# - homepage: http://openstack.org -# gerrit-host: review.openstack.org -# local-git-dir: /var/lib/git -# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key -# gerrit-committer: Project Creator -# has-github: True -# has-wiki: False -# has-issues: False -# has-downloads: False -# acl-dir: /home/gerrit2/acls -# acl-base: /home/gerrit2/acls/project.config -# --- -# - project: PROJECT_NAME -# options: -# - has-wiki -# - has-issues -# - has-downloads -# - has-pull-requests -# homepage: Some homepage that isn't http://openstack.org -# description: This is a great project -# remote: https://gerrit.googlesource.com/gerrit -# upstream: git://github.com/bushy/beards.git -# acl-config: /path/to/gerrit/project.config -# acl-append: -# - /path/to/gerrit/project.config -# acl-parameters: -# project: OTHER_PROJECT_NAME - -from __future__ import print_function - -import ConfigParser -import logging -import os -import re -import shlex -import subprocess -import tempfile -import yaml - -import gerritlib.gerrit -import github - -import jeepyb.gerritdb - -logging.basicConfig(level=logging.ERROR) -log = logging.getLogger("manage_projects") - - -def run_command(cmd, status=False, env={}): - cmd_list = shlex.split(str(cmd)) - newenv = os.environ - newenv.update(env) - log.debug("Executing command: %s" % " ".join(cmd_list)) - p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - log.debug("Return code: %s" % p.returncode) - log.debug("Command said: %s" % out.strip()) - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def run_command_status(cmd, env={}): - return run_command(cmd, True, env) - - -def git_command(repo_dir, sub_cmd, env={}): - git_dir = os.path.join(repo_dir, '.git') - cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) - status, _ = run_command(cmd, True, env) - return status - - -def git_command_output(repo_dir, sub_cmd, env={}): - git_dir = os.path.join(repo_dir, '.git') - cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) - status, out = run_command(cmd, True, env) - return (status, out) - - -def write_acl_config(project, acl_dir, acl_base, acl_append, parameters): - project_parts = os.path.split(project) - if len(project_parts) > 1: - repo_base = os.path.join(acl_dir, *project_parts[:-1]) - if not os.path.exists(repo_base): - os.makedirs(repo_base) - if not os.path.isdir(repo_base): - return 1 - project = project_parts[-1] - config_file = os.path.join(repo_base, "%s.config" % project) - else: - config_file = os.path.join(acl_dir, "%s.config" % project) - if 'project' not in parameters: - parameters['project'] = project - with open(config_file, 'w') as config: - if acl_base and os.path.exists(acl_base): - config.write(open(acl_base, 'r').read()) - for acl_snippet in acl_append: - if not os.path.exists(acl_snippet): - acl_snippet = os.path.join(acl_dir, acl_snippet) - if not os.path.exists(acl_snippet): - continue - with open(acl_snippet, 'r') as append_content: - config.write(append_content.read() % parameters) - - -def fetch_config(project, remote_url, repo_path, env={}): - status = git_command(repo_path, "fetch %s +refs/meta/config:" - "refs/remotes/gerrit-meta/config" % remote_url, env) - if status != 0: - print("Failed to fetch refs/meta/config for project: %s" % project) - return False - # Because the following fails if executed more than once you should only - # run fetch_config once in each repo. - status = git_command(repo_path, "checkout -b config " - "remotes/gerrit-meta/config") - if status != 0: - print("Failed to checkout config for project: %s" % project) - return False - - return True - - -def copy_acl_config(project, repo_path, acl_config): - if not os.path.exists(acl_config): - return False - - acl_dest = os.path.join(repo_path, "project.config") - status, _ = run_command("cp %s %s" % - (acl_config, acl_dest), status=True) - if status == 0: - status = git_command(repo_path, "diff --quiet") - if status != 0: - return True - return False - - -def push_acl_config(project, remote_url, repo_path, gitid, env={}): - cmd = "commit -a -m'Update project config.' --author='%s'" % gitid - status = git_command(repo_path, cmd) - if status != 0: - print("Failed to commit config for project: %s" % project) - return False - status, out = git_command_output(repo_path, - "push %s HEAD:refs/meta/config" % - remote_url, env) - if status != 0: - print("Failed to push config for project: %s" % project) - print(out) - return False - return True - - -def _get_group_uuid(gerrit, group): - cursor = jeepyb.gerritdb.connect().cursor() - query = "SELECT group_uuid FROM account_groups WHERE name = %s" - cursor.execute(query, group) - data = cursor.fetchone() - if data: - return data[0] - return None - - -def get_group_uuid(gerrit, group): - uuid = _get_group_uuid(gerrit, group) - if uuid: - return uuid - gerrit.createGroup(group) - uuid = _get_group_uuid(gerrit, group) - if uuid: - return uuid - return None - - -def create_groups_file(project, gerrit, repo_path): - acl_config = os.path.join(repo_path, "project.config") - group_file = os.path.join(repo_path, "groups") - uuids = {} - for line in open(acl_config, 'r'): - r = re.match(r'^\s+.*group\s+(.*)$', line) - if r: - group = r.group(1) - if group in uuids.keys(): - continue - uuid = get_group_uuid(gerrit, group) - if uuid: - uuids[group] = uuid - else: - return False - if uuids: - with open(group_file, 'w') as fp: - for group, uuid in uuids.items(): - fp.write("%s\t%s\n" % (uuid, group)) - status = git_command(repo_path, "add groups") - if status != 0: - print("Failed to add groups file for project: %s" % project) - return False - return True - - -def make_ssh_wrapper(gerrit_user, gerrit_key): - (fd, name) = tempfile.mkstemp(text=True) - os.write(fd, '#!/bin/bash\n') - os.write(fd, - 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' % - (gerrit_key, gerrit_user)) - os.close(fd) - os.chmod(name, 0o755) - return dict(GIT_SSH=name) - - -def create_github_project(defaults, options, project, description, homepage): - default_has_issues = defaults.get('has-issues', False) - default_has_downloads = defaults.get('has-downloads', False) - default_has_wiki = defaults.get('has-wiki', False) - has_issues = 'has-issues' in options or default_has_issues - has_downloads = 'has-downloads' in options or default_has_downloads - has_wiki = 'has-wiki' in options or default_has_wiki - - GITHUB_SECURE_CONFIG = defaults.get( - 'github-config', - '/etc/github/github-projects.secure.config') - - secure_config = ConfigParser.ConfigParser() - secure_config.read(GITHUB_SECURE_CONFIG) - - # Project creation doesn't work via oauth - ghub = github.Github(secure_config.get("github", "username"), - secure_config.get("github", "password")) - orgs = ghub.get_user().get_orgs() - orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) - - # Find the project's repo - project_split = project.split('/', 1) - org_name = project_split[0] - if len(project_split) > 1: - repo_name = project_split[1] - else: - repo_name = project - - try: - org = orgs_dict[org_name.lower()] - except KeyError: - # We do not have control of this github org ignore the project. - return - try: - repo = org.get_repo(repo_name) - except github.GithubException: - repo = org.create_repo(repo_name, - homepage=homepage, - has_issues=has_issues, - has_downloads=has_downloads, - has_wiki=has_wiki) - if description: - repo.edit(repo_name, description=description) - if homepage: - repo.edit(repo_name, homepage=homepage) - - repo.edit(repo_name, has_issues=has_issues, - has_downloads=has_downloads, - has_wiki=has_wiki) - - if 'gerrit' not in [team.name for team in repo.get_teams()]: - teams = org.get_teams() - teams_dict = dict(zip([t.name.lower() for t in teams], teams)) - teams_dict['gerrit'].add_to_repos(repo) - - -def main(): - - PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - configs = [config for config in yaml.load_all(open(PROJECTS_YAML))] - defaults = configs[0][0] - default_has_github = defaults.get('has-github', True) - - LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') - ACL_DIR = defaults.get('acl-dir') - GERRIT_HOST = defaults.get('gerrit-host') - GERRIT_PORT = int(defaults.get('gerrit-port', '29418')) - GERRIT_USER = defaults.get('gerrit-user') - GERRIT_KEY = defaults.get('gerrit-key') - GERRIT_GITID = defaults.get('gerrit-committer') - GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2') - GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2') - - gerrit = gerritlib.gerrit.Gerrit('localhost', - GERRIT_USER, - GERRIT_PORT, - GERRIT_KEY) - project_list = gerrit.listProjects() - ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) - try: - - for section in configs[1]: - project = section['project'] - options = section.get('options', dict()) - description = section.get('description', None) - homepage = section.get('homepage', defaults.get('homepage', None)) - upstream = section.get('upstream', None) - - project_git = "%s.git" % project - project_dir = os.path.join(LOCAL_GIT_DIR, project_git) - - if 'has-github' in options or default_has_github: - create_github_project(defaults, options, project, - description, homepage) - - remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project) - if project not in project_list: - tmpdir = tempfile.mkdtemp() - try: - repo_path = os.path.join(tmpdir, 'repo') - if upstream: - run_command("git clone %(upstream)s %(repo_path)s" % - dict(upstream=upstream, - repo_path=repo_path)) - git_command(repo_path, - "fetch origin " - "+refs/heads/*:refs/copy/heads/*", - env=ssh_env) - push_string = "push %s +refs/copy/heads/*:refs/heads/*" - else: - run_command("git init %s" % repo_path) - with open(os.path.join(repo_path, - ".gitreview"), - 'w') as gitreview: - gitreview.write("""[gerrit] -host=%s -port=%s -project=%s -""" % (GERRIT_HOST, GERRIT_PORT, project_git)) - git_command(repo_path, "add .gitreview") - cmd = ("commit -a -m'Added .gitreview' --author='%s'" - % GERRIT_GITID) - git_command(repo_path, cmd) - push_string = "push --all %s" - gerrit.createProject(project) - - if not os.path.exists(project_dir): - run_command("git --bare init %s" % project_dir) - run_command("chown -R %s:%s %s" - % (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP, - project_dir)) - - git_command(repo_path, - push_string % remote_url, - env=ssh_env) - git_command(repo_path, - "push --tags %s" % remote_url, env=ssh_env) - finally: - run_command("rm -fr %s" % tmpdir) - - try: - acl_config = section.get('acl-config', - '%s.config' % os.path.join(ACL_DIR, - project)) - except AttributeError: - acl_config = None - - if acl_config: - if not os.path.isfile(acl_config): - write_acl_config(project, - ACL_DIR, - section.get('acl-base', None), - section.get('acl-append', []), - section.get('acl-parameters', {})) - tmpdir = tempfile.mkdtemp() - try: - repo_path = os.path.join(tmpdir, 'repo') - ret, _ = run_command_status("git init %s" % repo_path) - if ret != 0: - continue - if (fetch_config(project, - remote_url, - repo_path, - ssh_env) and - copy_acl_config(project, repo_path, - acl_config) and - create_groups_file(project, gerrit, repo_path)): - push_acl_config(project, - remote_url, - repo_path, - GERRIT_GITID, - ssh_env) - finally: - run_command("rm -fr %s" % tmpdir) - finally: - os.unlink(ssh_env['GIT_SSH']) - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/notify_impact.py b/jeepyb/cmd/notify_impact.py deleted file mode 100644 index 93f2e10..0000000 --- a/jeepyb/cmd/notify_impact.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is designed to be called by a gerrit hook. It searched new -# patchsets for strings like "bug FOO" and updates corresponding Launchpad -# bugs status. - -import argparse -import os -import re -import smtplib -import subprocess - -from email.mime import text -from launchpadlib import launchpad -from launchpadlib import uris - -BASE_DIR = '/home/gerrit2/review_site' -EMAIL_TEMPLATE = """ -Hi, I'd like you to take a look at this patch for potential -%s. -%s - -Log: -%s -""" - -GERRIT_CACHE_DIR = os.path.expanduser( - os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser( - os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) - - -def create_bug(git_log, args, lp_project): - """Create a bug for a change. - - Create a launchpad bug in lp_project, titled with the first line of - the git commit message, with the content of the git_log prepended - with the Gerrit review URL. Tag the bug with the name of the repository - it came from. Don't create a duplicate bug. Returns link to the bug. - """ - lpconn = launchpad.Launchpad.login_with( - 'Gerrit User Sync', - uris.LPNET_SERVICE_ROOT, - GERRIT_CACHE_DIR, - credentials_file=GERRIT_CREDENTIALS, - version='devel') - - lines_in_log = git_log.split("\n") - bug_title = lines_in_log[4] - bug_descr = args.change_url + '\n' + git_log - project = lpconn.projects[lp_project] - # check for existing bugs by searching for the title, to avoid - # creating multiple bugs per review - potential_dupes = project.searchTasks(search_text=bug_title) - if len(potential_dupes) == 0: - buginfo = lpconn.bugs.createBug( - target=project, title=bug_title, - description=bug_descr, tags=args.project.split('/')[1]) - buglink = buginfo.web_link - - return buglink - - -def process_impact(git_log, args): - """Process DocImpact flag. - - If the 'DocImpact' flag is present, create a new documentation bug in - the openstack-manuals launchpad project based on the git_log. - For non-documentation impacts notify the mailing list of impact. - """ - if args.impact.lower() == 'docimpact': - create_bug(git_log, args, 'openstack-manuals') - return - - email_content = EMAIL_TEMPLATE % (args.impact, - args.change_url, git_log) - - msg = text.MIMEText(email_content) - msg['Subject'] = '[%s] %s review request change %s' % \ - (args.project, args.impact, args.change) - msg['From'] = 'gerrit2@review.openstack.org' - msg['To'] = args.dest_address - - s = smtplib.SMTP('localhost') - s.sendmail('gerrit2@review.openstack.org', - args.dest_address, msg.as_string()) - s.quit() - - -def impacted(git_log, impact_string): - """Determine if a changes log indicates there is an impact.""" - return re.search(impact_string, git_log, re.IGNORECASE) - - -def extract_git_log(args): - """Extract git log of all merged commits.""" - cmd = ['git', - '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', - 'log', '--no-merges', args.commit + '^1..' + args.commit] - return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('hook') - #common - parser.add_argument('--change', default=None) - parser.add_argument('--change-url', default=None) - parser.add_argument('--project', default=None) - parser.add_argument('--branch', default=None) - parser.add_argument('--commit', default=None) - #change-merged - parser.add_argument('--submitter', default=None) - #patchset-created - parser.add_argument('--uploader', default=None) - parser.add_argument('--patchset', default=None) - # Not passed by gerrit: - parser.add_argument('--impact', default=None) - parser.add_argument('--dest-address', default=None) - - args = parser.parse_args() - - # Get git log - git_log = extract_git_log(args) - - # Process impacts found in git log - if impacted(git_log, args.impact): - process_impact(git_log, args) - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/openstackwatch.py b/jeepyb/cmd/openstackwatch.py deleted file mode 100644 index 2381b4b..0000000 --- a/jeepyb/cmd/openstackwatch.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2013 Chmouel Boudjnah, eNovance -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This script is designed to generate rss feeds for subscription from updates -# to various gerrit tracked projects. It is intended to be run periodically, -# for example hourly via cron. It takes an optional argument to specify the -# path to a configuration file. -# -*- encoding: utf-8 -*- - -from __future__ import print_function - -__author__ = "Chmouel Boudjnah " - -import ConfigParser -import cStringIO -import datetime -import json -import os -import sys -import time -import urllib - -import PyRSS2Gen - -PROJECTS = ['openstack/nova', 'openstack/keystone', 'opensack/swift'] -JSON_URL = 'https://review.openstack.org/query' -DEBUG = False -OUTPUT_MODE = 'multiple' - -curdir = os.path.dirname(os.path.realpath(sys.argv[0])) - - -class ConfigurationError(Exception): - pass - - -def get_config(config, section, option, default=None): - if not config.has_section(section): - raise ConfigurationError("Invalid configuration, missing section: %s" % - section) - if config.has_option(section, option): - return config.get(section, option) - elif not default is None: - return default - else: - raise ConfigurationError("Invalid configuration, missing " - "section/option: %s/%s" % (section, option)) - - -def parse_ini(inifile): - ret = {} - if not os.path.exists(inifile): - return - config = ConfigParser.RawConfigParser(allow_no_value=True) - config.read(inifile) - - if config.has_section('swift'): - ret['swift'] = dict(config.items('swift')) - - ret['projects'] = get_config(config, 'general', 'projects', PROJECTS) - if type(ret['projects']) is not list: - ret['projects'] = [x.strip() for x in ret['projects'].split(',')] - ret['json_url'] = get_config(config, 'general', 'json_url', JSON_URL) - ret['debug'] = get_config(config, 'general', 'debug', DEBUG) - ret['output_mode'] = get_config(config, 'general', 'output_mode', - OUTPUT_MODE) - return ret - -try: - conffile = sys.argv[1] -except IndexError: - conffile = os.path.join(curdir, '..', 'config', 'openstackwatch.ini') -CONFIG = parse_ini(conffile) - - -def debug(msg): - if DEBUG: - print(msg) - - -def get_javascript(project=None): - url = CONFIG['json_url'] - if project: - url += "+project:" + project - fp = urllib.urlretrieve(url) - ret = open(fp[0]).read() - return ret - - -def parse_javascript(javascript): - for row in javascript.splitlines(): - try: - json_row = json.loads(row) - except(ValueError): - continue - if not json_row or not 'project' in json_row or \ - json_row['project'] not in CONFIG['projects']: - continue - yield json_row - - -def upload_rss(xml, output_object): - if 'swift' not in CONFIG: - print(xml) - return - - import swiftclient - cfg = CONFIG['swift'] - client = swiftclient.Connection(cfg['auth_url'], - cfg['username'], - cfg['password'], - auth_version=cfg.get('auth_version', - '2.0')) - try: - client.get_container(cfg['container']) - except(swiftclient.client.ClientException): - client.put_container(cfg['container']) - # eventual consistenties - time.sleep(1) - - client.put_object(cfg['container'], output_object, - cStringIO.StringIO(xml)) - - -def generate_rss(javascript, project=""): - title = "OpenStack %s watch RSS feed" % (project) - rss = PyRSS2Gen.RSS2( - title=title, - link="http://github.com/chmouel/openstackwatch.rss", - description="The latest reviews about Openstack, straight " - "from Gerrit.", - lastBuildDate=datetime.datetime.now() - ) - for row in parse_javascript(javascript): - author = row['owner']['name'] - author += " <%s>" % ('email' in row['owner'] and - row['owner']['email'] - or row['owner']['username']) - rss.items.append( - PyRSS2Gen.RSSItem( - title="%s [%s]: %s" % (os.path.basename(row['project']), - row['status'], - row['subject']), - author=author, - link=row['url'], - guid=PyRSS2Gen.Guid(row['id']), - description=row['subject'], - pubDate=datetime.datetime.fromtimestamp(row['lastUpdated']), - )) - return rss.to_xml() - - -def main(): - if CONFIG['output_mode'] == "combined": - upload_rss(generate_rss(get_javascript()), - CONFIG['swift']['combined_output_object']) - elif CONFIG['output_mode'] == "multiple": - for project in CONFIG['projects']: - upload_rss( - generate_rss(get_javascript(project), project=project), - "%s.xml" % (os.path.basename(project))) - -if __name__ == '__main__': - main() diff --git a/jeepyb/cmd/trivial_rebase.py b/jeepyb/cmd/trivial_rebase.py deleted file mode 100644 index 184ceb3..0000000 --- a/jeepyb/cmd/trivial_rebase.py +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright (c) 2010, Code Aurora Forum. All rights reserved. -# Copyright (c) 2012, Hewlett-Packard Development Company, L.P. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# # Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# # Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# # Neither the name of Code Aurora Forum, Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS -# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# This script is designed to detect when a patchset uploaded to Gerrit is -# 'identical' (determined via git-patch-id) and reapply reviews onto the new -# patchset from the previous patchset. - -# Get usage and help info by running: trivial-rebase --help -# Documentation is available here: -# https://www.codeaurora.org/xwiki/bin/QAEP/Gerrit - -from __future__ import print_function - -import json -import optparse -import subprocess -import sys - - -class SilentOptionParser(optparse.OptionParser): - """Make OptionParser silently swallow unrecognized options.""" - def _process_args(self, largs, rargs, values): - while rargs: - try: - optparse.OptionParser._process_args(self, largs, rargs, values) - except (optparse.AmbiguousOptionError, - optparse.BadOptionError) as e: - largs.append(e.opt_str) - - -class CheckCallError(OSError): - """CheckCall() returned non-0.""" - def __init__(self, command, cwd, retcode, stdout, stderr=None): - OSError.__init__(self, command, cwd, retcode, stdout, stderr) - self.command = command - self.cwd = cwd - self.retcode = retcode - self.stdout = stdout - self.stderr = stderr - - -def CheckCall(command, cwd=None): - """Like subprocess.check_call() but returns stdout. - - Works on python 2.4 - """ - try: - process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE) - std_out, std_err = process.communicate() - except OSError as e: - raise CheckCallError(command, cwd, e.errno, None) - if process.returncode: - raise CheckCallError(command, cwd, process.returncode, - std_out, std_err) - return std_out, std_err - - -def Gssh(options, api_command): - """Makes a Gerrit API call via SSH and returns the stdout results.""" - ssh_cmd = ['ssh', - '-l', 'Gerrit Code Review', - '-p', options.port, - '-i', options.private_key_path, - options.server, - api_command] - try: - return CheckCall(ssh_cmd)[0] - except CheckCallError as e: - err_template = "call: %s\nreturn code: %s\nstdout: %s\nstderr: %s\n" - sys.stderr.write(err_template % (ssh_cmd, - e.retcode, - e.stdout, - e.stderr)) - raise - - -def GsqlQuery(sql_query, options): - """Runs a gerrit gsql query and returns the result.""" - gsql_cmd = "gerrit gsql --format JSON -c %s" % sql_query - gsql_out = Gssh(options, gsql_cmd) - new_out = gsql_out.replace('}}\n', '}}\nsplit here\n') - return new_out.split('split here\n') - - -def FindPrevRev(options): - """Finds the revision of the previous patch set on the change.""" - sql_query = ("\"SELECT revision FROM patch_sets,changes WHERE " - "patch_sets.change_id = changes.change_id AND " - "patch_sets.patch_set_id = %s AND " - "changes.change_key = \'%s\'\"" % ((options.patchset - 1), - options.changeId)) - revisions = GsqlQuery(sql_query, options) - - json_dict = json.loads(revisions[0], strict=False) - return json_dict["columns"]["revision"] - - -def GetApprovals(options): - """Get all the approvals on a specific patch set. - - Returns a list of approval dicts - """ - sql_query = ("\"SELECT value,account_id,category_id" - " FROM patch_set_approvals" - " WHERE patch_set_id = %s" - " AND change_id = (SELECT change_id FROM" - " changes WHERE change_key = \'%s\') AND value <> 0\"" - % ((options.patchset - 1), options.changeId)) - gsql_out = GsqlQuery(sql_query, options) - approvals = [] - for json_str in gsql_out: - dict = json.loads(json_str, strict=False) - if dict["type"] == "row": - approvals.append(dict["columns"]) - return approvals - - -def GetPatchId(revision, consider_whitespace=False): - git_show_cmd = ['git', 'show', revision] - patch_id_cmd = ['git', 'patch-id'] - patch_id_process = subprocess.Popen(patch_id_cmd, stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - git_show_process = subprocess.Popen(git_show_cmd, stdout=subprocess.PIPE) - if consider_whitespace: - # This matches on change lines in the patch (those starting with "+" - # or "-" but not followed by another of the same), then replaces any - # space or tab characters with "%" before calculating a patch-id. - replace_ws_cmd = ['sed', r'/^\(+[^+]\|-[^-]\)/y/ \t/%%/'] - replace_ws_process = subprocess.Popen(replace_ws_cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - git_show_output = git_show_process.communicate()[0] - replace_ws_output = replace_ws_process.communicate(git_show_output)[0] - return patch_id_process.communicate(replace_ws_output)[0] - else: - return patch_id_process.communicate( - git_show_process.communicate()[0])[0] - - -def SuExec(options, as_user, cmd): - suexec_cmd = "suexec --as %s -- %s" % (as_user, cmd) - Gssh(options, suexec_cmd) - - -def DiffCommitMessages(commit1, commit2): - log_cmd1 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"', - commit1 + '^!'] - commit1_log = CheckCall(log_cmd1) - log_cmd2 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"', - commit2 + '^!'] - commit2_log = CheckCall(log_cmd2) - if commit1_log != commit2_log: - return True - return False - - -def main(): - usage = "usage: %prog [optional options]" - parser = SilentOptionParser(usage=usage) - parser.add_option("--change", dest="changeId", help="Change identifier") - parser.add_option("--project", help="Project path in Gerrit") - parser.add_option("--commit", help="Git commit-ish for this patchset") - parser.add_option("--patchset", type="int", help="The patchset number") - parser.add_option("--role-user", dest="role_user", - help="E-mail/ID of user commenting on commit messages") - parser.add_option("--private-key-path", dest="private_key_path", - help="Full path to Gerrit SSH daemon's private host key") - parser.add_option("--server-port", dest="port", default='29418', - help="Port to connect to Gerrit's SSH daemon " - "[default: %default]") - parser.add_option("--server", dest="server", default="localhost", - help="Server name/address for Gerrit's SSH daemon " - "[default: %default]") - parser.add_option("--whitespace", action="store_true", - help="Treat whitespace as significant") - - (options, args) = parser.parse_args() - - if not options.changeId: - parser.print_help() - sys.exit(0) - - if options.patchset == 1: - # Nothing to detect on first patchset - sys.exit(0) - prev_revision = None - prev_revision = FindPrevRev(options) - if not prev_revision: - # Couldn't find a previous revision - sys.exit(0) - prev_patch_id = GetPatchId(prev_revision) - cur_patch_id = GetPatchId(options.commit) - if cur_patch_id.split()[0] != prev_patch_id.split()[0]: - # patch-ids don't match - sys.exit(0) - # Patch ids match. This is a trivial rebase. - # In addition to patch-id we should check if whitespace content changed. - # Some languages are more sensitive to whitespace than others, and some - # changes may either introduce or be intended to fix style problems - # specifically involving whitespace as well. - if options.whitespace: - prev_patch_ws = GetPatchId(prev_revision, consider_whitespace=True) - cur_patch_ws = GetPatchId(options.commit, consider_whitespace=True) - if cur_patch_ws.split()[0] != prev_patch_ws.split()[0]: - # Insert a comment into the change letting the approvers know - # only the whitespace changed - comment_msg = ("\"New patchset patch-id matches previous patchset," - " but whitespace content has changed.\"") - comment_cmd = ['gerrit', 'approve', '--project', options.project, - '--message', comment_msg, options.commit] - SuExec(options, options.role_user, ' '.join(comment_cmd)) - sys.exit(0) - - # We should also check if the commit message changed. Most approvers would - # want to re-review changes when the commit message changes. - changed = DiffCommitMessages(prev_revision, options.commit) - if changed: - # Insert a comment into the change letting the approvers know only the - # commit message changed - comment_msg = ("\"New patchset patch-id matches previous patchset," - " but commit message has changed.\"") - comment_cmd = ['gerrit', 'approve', '--project', options.project, - '--message', comment_msg, options.commit] - SuExec(options, options.role_user, ' '.join(comment_cmd)) - sys.exit(0) - - # Need to get all approvals on prior patch set, then suexec them onto - # this patchset. - approvals = GetApprovals(options) - gerrit_approve_msg = ("\'Automatically re-added by Gerrit trivial rebase " - "detection script.\'") - for approval in approvals: - # Note: Sites with different 'copy_min_score' values in the - # approval_categories DB table might want different behavior here. - # Additional categories should also be added if desired. - if approval["category_id"] == "CRVW": - approve_category = '--code-review' - elif approval["category_id"] == "VRIF": - # Don't re-add verifies - #approve_category = '--verified' - continue - elif approval["category_id"] == "SUBM": - # We don't care about previous submit attempts - continue - elif approval["category_id"] == "APRV": - # Similarly squash old approvals - continue - else: - print("Unsupported category: %s" % approval) - sys.exit(0) - - score = approval["value"] - gerrit_approve_cmd = ['gerrit', 'approve', - '--project', options.project, - '--message', gerrit_approve_msg, - approve_category, score, options.commit] - SuExec(options, approval["account_id"], ' '.join(gerrit_approve_cmd)) - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/update_blueprint.py b/jeepyb/cmd/update_blueprint.py deleted file mode 100644 index 738ba13..0000000 --- a/jeepyb/cmd/update_blueprint.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is designed to be called by a gerrit hook. It searched new -# patchsets for strings like "blueprint FOO" or "bp FOO" and updates -# corresponding Launchpad blueprints with links back to the change. - -import argparse -import ConfigParser -import os -import re -import StringIO -import subprocess - -from launchpadlib import launchpad -from launchpadlib import uris -import MySQLdb - -BASE_DIR = '/home/gerrit2/review_site' -GERRIT_CACHE_DIR = os.path.expanduser( - os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser( - os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) -GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG', - '/home/gerrit2/review_site/etc/gerrit.config') -GERRIT_SECURE_CONFIG_DEFAULT = '/home/gerrit2/review_site/etc/secure.config' -GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG', - GERRIT_SECURE_CONFIG_DEFAULT) -SPEC_RE = re.compile(r'\b(blueprint|bp)\b[ \t]*[#:]?[ \t]*(\S+)', re.I) -BODY_RE = re.compile(r'^\s+.*$') - - -def get_broken_config(filename): - """gerrit config ini files are broken and have leading tabs.""" - text = "" - with open(filename, "r") as conf: - for line in conf.readlines(): - text = "%s%s" % (text, line.lstrip()) - - fp = StringIO.StringIO(text) - c = ConfigParser.ConfigParser() - c.readfp(fp) - return c - -GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG) -SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG) -DB_USER = GERRIT_CONFIG.get("database", "username") -DB_PASS = SECURE_CONFIG.get("database", "password") -DB_DB = GERRIT_CONFIG.get("database", "database") - - -def short_project(full_project_name): - """Return the project part of the git repository name.""" - return full_project_name.split('/')[-1] - - -def git2lp(full_project_name): - """Convert Git repo name to Launchpad project.""" - project_map = { - 'stackforge/puppet-openstack_dev_env': 'puppet-openstack', - 'stackforge/puppet-quantum': 'puppet-neutron', - } - return project_map.get(full_project_name, short_project(full_project_name)) - - -def update_spec(launchpad, project, name, subject, link, topic=None): - project = git2lp(project) - spec = launchpad.projects[project].getSpecification(name=name) - if not spec: - return - - if spec.whiteboard: - wb = spec.whiteboard.strip() - else: - wb = '' - changed = False - if topic: - topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/', 8)], - topic) - if topiclink not in wb: - wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink) - changed = True - - if link not in wb: - wb += ("\n\n\nAddressed by: {link}\n" - " {subject}\n").format(subject=subject, - link=link) - changed = True - - if changed: - spec.whiteboard = wb - spec.lp_save() - - -def find_specs(launchpad, dbconn, args): - git_dir_arg = '--git-dir={base_dir}/git/{project}.git'.format( - base_dir=BASE_DIR, - project=args.project) - git_log = subprocess.Popen(['git', git_dir_arg, 'log', '--no-merges', - args.commit + '^1..' + args.commit], - stdout=subprocess.PIPE).communicate()[0] - - cur = dbconn.cursor() - cur.execute("select subject, topic from changes where change_key=%s", - args.change) - subject, topic = cur.fetchone() - specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)]) - - if topic: - topicspec = topic.split('/')[-1] - specs |= set([topicspec]) - - for spec in specs: - update_spec(launchpad, args.project, spec, subject, - args.change_url, topic) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('hook') - #common - parser.add_argument('--change', default=None) - parser.add_argument('--change-url', default=None) - parser.add_argument('--project', default=None) - parser.add_argument('--branch', default=None) - parser.add_argument('--commit', default=None) - #change-merged - parser.add_argument('--submitter', default=None) - # patchset-created - parser.add_argument('--uploader', default=None) - parser.add_argument('--patchset', default=None) - - args = parser.parse_args() - - lpconn = launchpad.Launchpad.login_with( - 'Gerrit User Sync', uris.LPNET_SERVICE_ROOT, GERRIT_CACHE_DIR, - credentials_file=GERRIT_CREDENTIALS, version='devel') - - conn = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB) - - find_specs(lpconn, conn, args) - -if __name__ == "__main__": - main() diff --git a/jeepyb/cmd/update_bug.py b/jeepyb/cmd/update_bug.py deleted file mode 100644 index 91c674f..0000000 --- a/jeepyb/cmd/update_bug.py +++ /dev/null @@ -1,405 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is designed to be called by a gerrit hook. It searched new -# patchsets for strings like "bug FOO" and updates corresponding Launchpad -# bugs status. - -import argparse -import os -import re -import subprocess - -from launchpadlib import launchpad -from launchpadlib import uris - -import jeepyb.gerritdb - -BASE_DIR = '/home/gerrit2/review_site' -GERRIT_CACHE_DIR = os.path.expanduser( - os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser( - os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) - - -def add_change_proposed_message(bugtask, change_url, project, branch): - subject = 'Fix proposed to %s (%s)' % (short_project(project), branch) - body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url) - bugtask.bug.newMessage(subject=subject, content=body) - - -def add_change_merged_message(bugtask, change_url, project, commit, - submitter, branch, git_log): - subject = 'Fix merged to %s (%s)' % (short_project(project), branch) - git_url = 'http://github.com/%s/commit/%s' % (project, commit) - body = '''Reviewed: %s -Committed: %s -Submitter: %s -Branch: %s\n''' % (change_url, git_url, submitter, branch) - body = body + '\n' + git_log - bugtask.bug.newMessage(subject=subject, content=body) - - -def set_in_progress(bugtask, launchpad, uploader, change_url): - """Set bug In progress with assignee being the uploader""" - - # Retrieve uploader from Launchpad by correlating Gerrit E-mail - # address to OpenID, and only set if there is a clear match. - try: - searchkey = uploader[uploader.rindex("(") + 1:-1] - except ValueError: - searchkey = uploader - - # The counterintuitive query is due to odd database schema choices - # in Gerrit. For example, an account with a secondary E-mail - # address added looks like... - # select email_address,external_id from account_external_ids - # where account_id=1234; - # +-----------------+-----------------------------------------+ - # | email_address | external_id | - # +-----------------+-----------------------------------------+ - # | plugh@xyzzy.com | https://login.launchpad.net/+id/fR0bnU1 | - # | bar@foo.org | mailto:bar@foo.org | - # | NULL | username:quux | - # +-----------------+-----------------------------------------+ - # ...thus we need a join on a secondary query to search against - # all the user's configured E-mail addresses. - # - query = """SELECT t.external_id FROM account_external_ids t - INNER JOIN ( - SELECT t.account_id FROM account_external_ids t - WHERE t.email_address = %s ) - original ON t.account_id = original.account_id - AND t.external_id LIKE 'https://login.launchpad.net%%'""" - - cursor = jeepyb.gerritdb.connect().cursor() - cursor.execute(query, searchkey) - data = cursor.fetchone() - if data: - assignee = launchpad.people.getByOpenIDIdentifier(identifier=data[0]) - if assignee: - bugtask.assignee = assignee - - bugtask.status = "In Progress" - bugtask.lp_save() - - -def set_fix_committed(bugtask): - """Set bug fix committed.""" - - bugtask.status = "Fix Committed" - bugtask.lp_save() - - -def set_fix_released(bugtask): - """Set bug fix released.""" - - bugtask.status = "Fix Released" - bugtask.lp_save() - - -def release_fixcommitted(bugtask): - """Set bug FixReleased if it was FixCommitted.""" - - if bugtask.status == u'Fix Committed': - set_fix_released(bugtask) - - -def tag_in_branchname(bugtask, branch): - """Tag bug with in-branch-name tag (if name is appropriate).""" - - lp_bug = bugtask.bug - branch_name = branch.replace('/', '-') - if branch_name.replace('-', '').isalnum(): - lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name] - lp_bug.tags.append("in-%s" % branch_name) - lp_bug.lp_save() - - -def short_project(full_project_name): - """Return the project part of the git repository name.""" - return full_project_name.split('/')[-1] - - -def git2lp(full_project_name): - """Convert Git repo name to Launchpad project.""" - project_map = { - 'openstack/api-site': 'openstack-api-site', - 'openstack/quantum': 'neutron', - 'openstack/python-quantumclient': 'python-neutronclient', - 'openstack/oslo-incubator': 'oslo', - 'openstack/tripleo-incubator': 'tripleo', - 'openstack-infra/askbot-theme': 'openstack-ci', - 'openstack-infra/config': 'openstack-ci', - 'openstack-infra/devstack-gate': 'openstack-ci', - 'openstack-infra/gear': 'openstack-ci', - 'openstack-infra/gerrit': 'openstack-ci', - 'openstack-infra/gerritbot': 'openstack-ci', - 'openstack-infra/gerritlib': 'openstack-ci', - 'openstack-infra/gitdm': 'openstack-ci', - 'openstack-infra/jeepyb': 'openstack-ci', - 'openstack-infra/jenkins-job-builder': 'openstack-ci', - 'openstack-infra/lodgeit': 'openstack-ci', - 'openstack-infra/meetbot': 'openstack-ci', - 'openstack-infra/nose-html-output': 'openstack-ci', - 'openstack-infra/publications': 'openstack-ci', - 'openstack-infra/puppet-apparmor': 'openstack-ci', - 'openstack-infra/puppet-dashboard': 'openstack-ci', - 'openstack-infra/puppet-vcsrepo': 'openstack-ci', - 'openstack-infra/reviewday': 'openstack-ci', - 'openstack-infra/statusbot': 'openstack-ci', - 'openstack-infra/zmq-event-publisher': 'openstack-ci', - 'stackforge/cookbook-openstack-block-storage': 'openstack-chef', - 'stackforge/cookbook-openstack-common': 'openstack-chef', - 'stackforge/cookbook-openstack-compute': 'openstack-chef', - 'stackforge/cookbook-openstack-dashboard': 'openstack-chef', - 'stackforge/cookbook-openstack-identity': 'openstack-chef', - 'stackforge/cookbook-openstack-image': 'openstack-chef', - 'stackforge/cookbook-openstack-metering': 'openstack-chef', - 'stackforge/cookbook-openstack-network': 'openstack-chef', - 'stackforge/cookbook-openstack-object-storage': 'openstack-chef', - 'stackforge/cookbook-openstack-ops-database': 'openstack-chef', - 'stackforge/cookbook-openstack-ops-messaging': 'openstack-chef', - 'stackforge/cookbook-openstack-orchestration': 'openstack-chef', - 'stackforge/openstack-chef-repo': 'openstack-chef', - 'stackforge/puppet-openstack_dev_env': 'puppet-openstack', - 'stackforge/puppet-quantum': 'puppet-neutron', - 'stackforge/tripleo-heat-templates': 'tripleo', - 'stackforge/tripleo-image-elements': 'tripleo', - } - return project_map.get(full_project_name, short_project(full_project_name)) - - -def is_direct_release(full_project_name): - """Test against a list of projects who directly release changes.""" - return full_project_name in [ - 'openstack/openstack-manuals', - 'openstack/api-site', - 'openstack/tripleo-incubator', - 'openstack-dev/devstack', - 'openstack-infra/askbot-theme', - 'openstack-infra/config', - 'openstack-infra/devstack-gate', - 'openstack-infra/gerrit', - 'openstack-infra/gerritbot', - 'openstack-infra/gerritlib', - 'openstack-infra/gitdm', - 'openstack-infra/lodgeit', - 'openstack-infra/meetbot', - 'openstack-infra/nose-html-output', - 'openstack-infra/publications', - 'openstack-infra/reviewday', - 'openstack-infra/statusbot', - 'stackforge/cookbook-openstack-block-storage', - 'stackforge/cookbook-openstack-common', - 'stackforge/cookbook-openstack-compute', - 'stackforge/cookbook-openstack-dashboard', - 'stackforge/cookbook-openstack-identity', - 'stackforge/cookbook-openstack-image', - 'stackforge/cookbook-openstack-metering', - 'stackforge/cookbook-openstack-network', - 'stackforge/cookbook-openstack-object-storage', - 'stackforge/cookbook-openstack-ops-database', - 'stackforge/cookbook-openstack-ops-messaging', - 'stackforge/cookbook-openstack-orchestration', - 'stackforge/openstack-chef-repo', - 'stackforge/tripleo-heat-templates', - 'stackforge/tripleo-image-elements', - ] - - -class Task: - def __init__(self, lp_task, prefix): - '''Prefixes associated with bug references will allow for certain - changes to be made to the bug's launchpad (lp) page. The following - tokens represent the automation currently taking place. - - :: - add_comment -> Adds a comment to the bug's lp page. - set_in_progress -> Sets the bug's lp status to 'In Progress'. - set_fix_released -> Sets the bug's lp status to 'Fix Released'. - set_fix_committed -> Sets the bug's lp status to 'Fix Committed'. - :: - - changes_needed, when populated, simply indicates the actions that are - available to be taken based on the value of 'prefix'. - ''' - self.lp_task = lp_task - self.changes_needed = [] - - # If no prefix was matched, default to 'closes'. - prefix = prefix.split('-')[0].lower() if prefix else 'closes' - - if prefix in ('closes', 'fixes', 'resolves'): - self.changes_needed.extend(('add_comment', - 'set_in_progress', - 'set_fix_committed', - 'set_fix_released')) - elif prefix in ('partial',): - self.changes_needed.extend(('add_comment', 'set_in_progress')) - elif prefix in ('related', 'impacts', 'affects'): - self.changes_needed.extend(('add_comment',)) - else: - # prefix is not recognized. - self.changes_needed.extend(('add_comment',)) - - def needs_change(self, change): - '''Return a boolean indicating if given 'change' needs to be made.''' - if change in self.changes_needed: - return True - else: - return False - - -def process_bugtask(launchpad, task, git_log, args): - """Apply changes to lp bug tasks, based on hook / branch.""" - - bugtask = task.lp_task - - if args.hook == "change-merged": - if args.branch == 'master': - if (is_direct_release(args.project) and - task.needs_change('set_fix_released')): - set_fix_released(bugtask) - else: - if (bugtask.status != u'Fix Released' and - task.needs_change('set_fix_committed')): - set_fix_committed(bugtask) - elif args.branch == 'milestone-proposed': - release_fixcommitted(bugtask) - elif args.branch.startswith('stable/'): - series = args.branch[7:] - # Look for a related task matching the series. - for reltask in bugtask.related_tasks: - if (reltask.bug_target_name.endswith("/" + series) and - reltask.status != u'Fix Released' and - task.needs_change('set_fix_committed')): - set_fix_committed(reltask) - break - else: - # Use tag_in_branchname if there isn't any. - tag_in_branchname(bugtask, args.branch) - - add_change_merged_message(bugtask, args.change_url, args.project, - args.commit, args.submitter, args.branch, - git_log) - - if args.hook == "patchset-created": - if args.branch == 'master': - if (bugtask.status not in [u'Fix Committed', u'Fix Released'] and - task.needs_change('set_in_progress')): - set_in_progress(bugtask, launchpad, - args.uploader, args.change_url) - elif args.branch.startswith('stable/'): - series = args.branch[7:] - for reltask in bugtask.related_tasks: - if (reltask.bug_target_name.endswith("/" + series) and - task.needs_change('set_in_progress') and - reltask.status not in [u'Fix Committed', - u'Fix Released']): - set_in_progress(reltask, launchpad, - args.uploader, args.change_url) - break - - if args.patchset == '1': - add_change_proposed_message(bugtask, args.change_url, - args.project, args.branch) - - -def find_bugs(launchpad, git_log, args): - '''Find bugs referenced in the git log and return related tasks. - - Our regular expression is composed of three major parts: - part1: Matches only at start-of-line (required). Optionally matches any - word or hyphen separated words. - part2: Matches the words 'bug' or 'lp' on a word boundry (required). - part3: Matches a whole number (required). - - The following patterns will be matched properly: - bug # 555555 - Closes-Bug: 555555 - Fixes: bug # 555555 - Resolves: bug 555555 - Partial-Bug: lp bug # 555555 - - :returns: an iterable containing Task objects. - ''' - - part1 = r'^[\t ]*(?P[-\w]+)?[\s:]*' - part2 = r'(?:\b(?:bug|lp)\b[\s#:]*)+' - part3 = r'(?P\d+)\s*?$' - regexp = part1 + part2 + part3 - matches = re.finditer(regexp, git_log, flags=re.I | re.M) - - # Extract unique bug tasks and associated prefixes. - bugtasks = {} - for match in matches: - prefix = match.group('prefix') - bug_num = match.group('bug_number') - if bug_num not in bugtasks: - try: - lp_bug = launchpad.bugs[bug_num] - for lp_task in lp_bug.bug_tasks: - if lp_task.bug_target_name == git2lp(args.project): - bugtasks[bug_num] = Task(lp_task, prefix) - break - except KeyError: - # Unknown bug. - pass - - return bugtasks.values() - - -def extract_git_log(args): - """Extract git log of all merged commits.""" - cmd = ['git', - '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', - 'log', '--no-merges', args.commit + '^1..' + args.commit] - return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('hook') - # common - parser.add_argument('--change', default=None) - parser.add_argument('--change-url', default=None) - parser.add_argument('--project', default=None) - parser.add_argument('--branch', default=None) - parser.add_argument('--commit', default=None) - # change-merged - parser.add_argument('--submitter', default=None) - # patchset-created - parser.add_argument('--uploader', default=None) - parser.add_argument('--patchset', default=None) - - args = parser.parse_args() - - # Connect to Launchpad. - lpconn = launchpad.Launchpad.login_with( - 'Gerrit User Sync', uris.LPNET_SERVICE_ROOT, GERRIT_CACHE_DIR, - credentials_file=GERRIT_CREDENTIALS, version='devel') - - # Get git log. - git_log = extract_git_log(args) - - # Process tasks found in git log. - for task in find_bugs(lpconn, git_log, args): - process_bugtask(lpconn, task, git_log, args) - -if __name__ == "__main__": - main() diff --git a/jeepyb/config/openstackwatch.ini-sample b/jeepyb/config/openstackwatch.ini-sample deleted file mode 100644 index bf13234..0000000 --- a/jeepyb/config/openstackwatch.ini-sample +++ /dev/null @@ -1,39 +0,0 @@ -# -*- Mode: conf -*- - -[general] -# only show certain projects (don't forget the openstack/ as start) -projects = openstack/swift, openstack/cinder - -# The Json URL where is the gerrit system. -json_url = https://review.openstack.org/query?q=status:open - -# Allow different mode to output to swift, by default 'combined' will -# combined all rss in one and 'multiple' will upload all the projects -# in each rss file. -output_mode = multiple - -# username to your swift cluster -# [swift] -# username/tenant for swift with 2.0 or just username with 1.0 (i.e: -# RAX). -# username = - -# passowrd or api key -# password = - -# container to upload (probably want to be public) -# container = - -# auth_url of the cluster, for Rackspace this is : -# https://auth.api.rackspacecloud.com/v1.0 -# or Rackspace UK : -# https://lon.auth.api.rackspacecloud.com/v1.0 -# auth_url = https://lon.auth.api.rackspacecloud.com/v1.0 - -# auth version (1.0 for Rackspace clouds, 2.0 for keystone backend clusters) -# auth_version = 1.0 - -# the object name where to store the combined rss -# combined_output_object = openstackwatch.xml - -# vim: ft=dosini diff --git a/jeepyb/gerritdb.py b/jeepyb/gerritdb.py deleted file mode 100644 index c3be3d7..0000000 --- a/jeepyb/gerritdb.py +++ /dev/null @@ -1,55 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# Copyright (c) 2012 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import ConfigParser -import MySQLdb -import os -import StringIO - - -GERRIT_CONFIG = os.environ.get( - 'GERRIT_CONFIG', - '/home/gerrit2/review_site/etc/gerrit.config') -GERRIT_SECURE_CONFIG = os.environ.get( - 'GERRIT_SECURE_CONFIG', - '/home/gerrit2/review_site/etc/secure.config') -db_connection = None - - -def get_broken_config(filename): - """gerrit config ini files are broken and have leading tabs.""" - text = "" - for line in open(filename, "r"): - text += line.lstrip() - - fp = StringIO.StringIO(text) - c = ConfigParser.ConfigParser() - c.readfp(fp) - return c - - -def connect(): - global db_connection - if not db_connection: - gerrit_config = get_broken_config(GERRIT_CONFIG) - secure_config = get_broken_config(GERRIT_SECURE_CONFIG) - - DB_USER = gerrit_config.get("database", "username") - DB_PASS = secure_config.get("database", "password") - DB_DB = gerrit_config.get("database", "database") - - db_connection = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB) - return db_connection diff --git a/jeepyb/__init__.py b/pypi_mirror/__init__.py similarity index 100% rename from jeepyb/__init__.py rename to pypi_mirror/__init__.py diff --git a/jeepyb/cmd/__init__.py b/pypi_mirror/cmd/__init__.py similarity index 100% rename from jeepyb/cmd/__init__.py rename to pypi_mirror/cmd/__init__.py diff --git a/jeepyb/cmd/run_mirror.py b/pypi_mirror/cmd/run_mirror.py similarity index 100% rename from jeepyb/cmd/run_mirror.py rename to pypi_mirror/cmd/run_mirror.py diff --git a/requirements.txt b/requirements.txt index d223638..9666ba5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,3 @@ -gerritlib -MySQL-python -paramiko -PyGithub -pyyaml +PyYAML>=3.1.0 pkginfo -PyRSS2Gen -python-swiftclient pip>=1.4 diff --git a/setup.cfg b/setup.cfg index 4673363..350db75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] -name = jeepyb -summary = Tools for managing gerrit projects and external sources. +name = pypi-mirror +summary = Utility to build a partial PyPI mirror description-file = README.rst author = OpenStack Infrastructure Team @@ -16,21 +16,6 @@ classifier = Programming Language :: Python :: 2.7 Programming Language :: Python :: 2.6 -[global] -setup-hooks = - pbr.hooks.setup_hook - [entry_points] console_scripts = - close-pull-requests = jeepyb.cmd.close_pull_requests:main - create-cgitrepos = jeepyb.cmd.create_cgitrepos:main - expire-old-reviews = jeepyb.cmd.expire_old_reviews:main - fetch-remotes = jeepyb.cmd.fetch_remotes:main - manage-projects = jeepyb.cmd.manage_projects:main - notify-impact = jeepyb.cmd.notify_impact:main - openstackwatch = jeepyb.cmd.openstackwatch:main - process-cache = jeepyb.cmd.process_cache:main - run-mirror = jeepyb.cmd.run_mirror:main - trivial-rebase = jeepyb.cmd.trivial_rebase:main - update-blueprint = jeepyb.cmd.update_blueprint:main - update-bug = jeepyb.cmd.update_bug:main + run-mirror = pypi_mirror.cmd.run_mirror:main diff --git a/setup.py b/setup.py index 59a0090..70c2b3f 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools setuptools.setup( - setup_requires=['d2to1', 'pbr'], - d2to1=True) + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt index 974f05f..f0dd00b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1 +1 @@ -hacking>=0.5.6,<0.7 +hacking>=0.5.6,<0.8