x/anvil
Code Issues Proposed changes

Create a separate tool for generating git changelogs

Browse Source 
Change-Id: Ide0aed0b16ea06e8e0368ad1352a7808a6fef35f
This commit is contained in:
Alessio Ababilov 2013-06-06 07:09:41 +04:00 committed by Joshua Harlow
parent 27f9e46fcf
commit 06fc741026
3 changed files with 253 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tools/README.rst
View File

@ -10,30 +10,30 @@ multipip
Double requirement given: nose>=2 (already in nose>=1.2, name='nose')
Use `multipip` to join these requirements::
$ multipip 'nose>=1.2' 'nose>=2' 'nose<4'
{"compatibles": ["nose>=2,<4"], "incompatibles": {}}
Files of requirements can be used as well::
$ cat pip-requires
$ cat pip-requires
nose<4
$ multipip 'nose>=1.2' 'nose>=2' -r pip-requires
$ multipip 'nose>=1.2' 'nose>=2' -r pip-requires
{"compatibles": ["nose>=2,<4"], "incompatibles": {}}
`multipip` prints error messages for badly formated requirements and exits early
and for incompatible requirements provides you which package was incompatible
and which versions were found to be problematic::
$ cat pip-requires
$ cat pip-requires
pip==1.3
$ multipip 'pip==1.2' -r pip-requires
$ multipip 'pip==1.2' -r pip-requires
{"compatibles": [], "incompatibles": {"pip": ["pip==1.2", "pip==1.3"]}}
It is possible to filter some packages from printed output. This can
be useful for a huge `pip-requires` file::
$ cat pip-requires
$ cat pip-requires
nose<4
pip==1.2
nose>=1.2
@ -43,7 +43,7 @@ be useful for a huge `pip-requires` file::
Installed packages can be filtered, too (they are taken from `pip
freeze`)::
$ cat pip-requires
$ cat pip-requires
nose<4
pip==1.2
nose>=1.2
@ -83,6 +83,12 @@ builds RPMs (current directory is used by default)::
Wrote: /home/guest/rpmbuild/RPMS/noarch/python-multipip-0.1-1.noarch.rpm
...
git-changelog
-------------
This tool generates a pretty software's changelog from git history.
build-install-node-from-source.sh
---------------------------------

237
tools/git-changelog Executable file
View File

@ -0,0 +1,237 @@
#!/usr/bin/python
"""
This tool generates a pretty software's changelog from git history.
http://fedoraproject.org/wiki/How_to_create_an_RPM_package says:
%changelog: Changes in the package. Use the format example above.
Do NOT put software's changelog at here. This changelog
is for RPM itself.
"""
import argparse
import collections
import iso8601
import logging
import re
import os
import os.path
import subprocess
import sys
import textwrap
logger = logging.getLogger()
per_call_am = 50
class ExecutionError(Exception):
pass
def translate_utf8(text):
return text.decode('utf8').encode('ascii', 'replace')
def parse_mailmap(wkdir):
mapping = {}
mailmap_fn = os.path.join(wkdir, '.mailmap')
if not os.path.isfile(mailmap_fn):
return mapping
for line in open(mailmap_fn, 'rb').read().splitlines():
line = line.strip()
if len(line) and not line.startswith('#') and ' ' in line:
try:
(canonical_email, alias) = [x for x in line.split(' ')
if x.startswith('<')]
mapping[alias] = canonical_email
except (TypeError, ValueError, IndexError):
pass
return mapping
# Based off of http://www.brianlane.com/nice-changelog-entries.html
class GitChangeLog(object):
def __init__(self, wkdir):
self.wkdir = wkdir
self.date_buckets = None
def _get_commit_detail(self, commit, field, am=1):
detail_cmd = ['git', 'log', '--color=never', '-%s' % (am),
"--pretty=format:%s" % (field), commit]
(stdout, _stderr) = call_subprocess(detail_cmd, cwd=self.wkdir,
show_stdout=False)
ret = stdout.strip('\n').splitlines()
if len(ret) == 1:
ret = ret[0]
else:
ret = [x for x in ret if x.strip() != '']
ret = "\n".join(ret)
return ret
def get_log(self, commit):
if self.date_buckets is None:
self.date_buckets = self._get_log(commit)
return self.date_buckets
def _skip_entry(self, summary, date, email, name):
for f in [summary, name, email]:
try:
translate_utf8(f)
except UnicodeError:
logger.warn("Non-utf8 field %s found", f)
return True
email = email.lower().strip()
summary = summary.strip()
if not all([summary, date, email, name]):
return True
return False
def _get_log(self, commit):
log_cmd = ['git', 'log',
'--no-merges', '--pretty=oneline',
'--color=never', commit]
(sysout, _stderr) = call_subprocess(log_cmd, cwd=self.wkdir,
show_stdout=False)
lines = sysout.strip('\n').splitlines()
# Extract the raw commit details
mailmap = parse_mailmap(self.wkdir)
log = []
for i in range(0, len(lines), per_call_am):
line = lines[i]
fields = line.split(' ')
if not len(fields):
continue
# See: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt
commit_id = fields[0]
commit_details = self._get_commit_detail(commit_id,
"[%s][%ai][%aE][%an]",
per_call_am)
# Extracts the pieces that should be in brackets.
details_matcher = r"^\s*\[(.*?)\]\[(.*?)\]\[(.*?)\]\[(.*?)\]\s*$"
for a_commit in commit_details.splitlines():
matcher = re.match(details_matcher, a_commit)
if not matcher:
continue
(summary, date, author_email, author_name) = matcher.groups()
author_email = mailmap.get(author_email, author_email)
try:
date = iso8601.parse_date(date)
except iso8601.ParseError:
date = None
if self._skip_entry(summary, date, author_email, author_name):
continue
log.append({
'summary': translate_utf8(summary),
'when': date,
'author_email': translate_utf8(author_email),
'author_name': translate_utf8(author_name),
})
# Bucketize the dates by day
date_buckets = collections.defaultdict(list)
for entry in log:
day = entry['when'].date()
date_buckets[day].append(entry)
return date_buckets
def format_log(self, commit):
date_buckets = self.get_log(commit)
lines = []
for d in reversed(sorted(date_buckets.keys())):
entries = date_buckets[d]
for entry in entries:
header = "* %s %s <%s>" % (d.strftime("%a %b %d %Y"),
entry['author_name'],
entry['author_email'])
lines.append(header)
summary = entry['summary']
sublines = textwrap.wrap(summary, 77)
if len(sublines):
lines.append("- %s" % sublines[0])
if len(sublines) > 1:
for subline in sublines[1:]:
lines.append(" %s" % subline)
lines.append("")
return "\n".join(lines)
def create_parser():
parser = argparse.ArgumentParser()
parser.add_argument(
"--debug", "-d",
action="store_true",
default=False,
help="Print debug information")
parser.add_argument(
"--filename", "-f",
default="ChangeLog",
help="Name of changelog file (default: ChangeLog)")
parser.add_argument(
"commit",
metavar="<commit>",
default="HEAD",
nargs="?",
help="The name of a commit for which to generate the log"
" (default: HEAD)")
return parser
def call_subprocess(cmd, cwd=None, show_stdout=True, raise_on_returncode=True):
if show_stdout:
stdout = None
else:
stdout = subprocess.PIPE
proc = subprocess.Popen(cmd, cwd=cwd, stderr=None, stdin=None, stdout=stdout)
ret = proc.communicate()
if proc.returncode:
cwd = cwd or os.getcwd()
command_desc = " ".join(cmd)
if raise_on_returncode:
raise ExecutionError(
"Command %s failed with error code %s in %s"
% (command_desc, proc.returncode, cwd))
else:
logger.warn(
"Command %s had error code %s in %s"
% (command_desc, proc.returncode, cwd))
return ret
def setup_logging(options):
level = logging.DEBUG if options.debug else logging.WARNING
handler = logging.StreamHandler(sys.stderr)
logger.addHandler(handler)
logger.setLevel(level)
def main():
parser = create_parser()
options = parser.parse_args()
setup_logging(options)
source_dir = os.getcwd()
# .git can be a dir or a gitref regular file (for a git submodule)
if not os.path.exists(os.path.join(source_dir, ".git")):
print >> sys.stderr, "fatal: Not a git repository"
sys.exit(1)
try:
with open("%s/%s" % (source_dir, options.filename), "wb") as out:
out.write(GitChangeLog(source_dir).format_log(options.commit))
except Exception as ex:
print >> sys.stderr, ex
if __name__ == "__main__":
try:
main()
except Exception as exp:
print >> sys.stderr, exp

179
tools/py2rpm
View File

@ -1,9 +1,7 @@
#!/usr/bin/python
import argparse
import collections
import distutils.spawn
import iso8601
import logging
import re
import os
@ -12,7 +10,6 @@ import shutil
import subprocess
import sys
import tempfile
import textwrap
import pip.util
import pkg_resources
@ -49,22 +46,8 @@ arch_dependent = [
epoch_map = {}
skip_emails = [
'jenkins@review.openstack.org',
]
requirements_section_re = re.compile(r'\[(.*?)\]')
version_re = re.compile(r"^(.*[^.0])(\.0+)*$")
skip_summaries = [
re.compile(r'^merge commit', re.I),
re.compile(r'^merge branch', re.I),
re.compile(r'^merge pull', re.I),
re.compile(r'^merge remote', re.I),
]
per_call_am = 50
setup_py = "setup.py"
@ -72,145 +55,6 @@ class InstallationError(Exception):
pass
def translate_utf8(text):
return text.decode('utf8').encode('ascii', 'replace')
def parse_mailmap(wkdir):
mapping = {}
mailmap_fn = os.path.join(wkdir, '.mailmap')
if not os.path.isfile(mailmap_fn):
return mapping
for line in open(mailmap_fn, 'rb').read().splitlines():
line = line.strip()
if len(line) and not line.startswith('#') and ' ' in line:
try:
(canonical_email, alias) = [x for x in line.split(' ')
if x.startswith('<')]
mapping[alias] = canonical_email
except (TypeError, ValueError, IndexError):
pass
return mapping
# Based off of http://www.brianlane.com/nice-changelog-entries.html
class GitChangeLog(object):
def __init__(self, wkdir):
self.wkdir = wkdir
self.date_buckets = None
@staticmethod
def can_build_for(wkdir):
if os.path.isdir(os.path.join(wkdir, ".git")):
return True
return False
def _get_commit_detail(self, commit, field, am=1):
detail_cmd = ['git', 'log', '--color=never', '-%s' % (am),
"--pretty=format:%s" % (field), commit]
(stdout, _stderr) = call_subprocess(detail_cmd, cwd=self.wkdir,
show_stdout=False)
ret = stdout.strip('\n').splitlines()
if len(ret) == 1:
ret = ret[0]
else:
ret = [x for x in ret if x.strip() != '']
ret = "\n".join(ret)
return ret
def get_log(self):
if self.date_buckets is None:
self.date_buckets = self._get_log()
return self.date_buckets
def _skip_entry(self, summary, date, email, name):
for f in [summary, name, email]:
try:
translate_utf8(f)
except UnicodeError:
logger.warn("Non-utf8 field %s found", f)
return True
email = email.lower().strip()
if email in skip_emails:
return True
summary = summary.strip()
for r in skip_summaries:
if r.search(summary):
return True
if not all([summary, date, email, name]):
return True
return False
def _get_log(self):
log_cmd = ['git', 'log', '--pretty=oneline', '--color=never']
(sysout, _stderr) = call_subprocess(log_cmd, cwd=self.wkdir,
show_stdout=False)
lines = sysout.strip('\n').splitlines()
# Extract the raw commit details
mailmap = parse_mailmap(self.wkdir)
log = []
for i in range(0, len(lines), per_call_am):
line = lines[i]
fields = line.split(' ')
if not len(fields):
continue
# See: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt
commit_id = fields[0]
commit_details = self._get_commit_detail(commit_id,
"[%s][%ai][%aE][%an]",
per_call_am)
# Extracts the pieces that should be in brackets.
details_matcher = r"^\s*\[(.*?)\]\[(.*?)\]\[(.*?)\]\[(.*?)\]\s*$"
for a_commit in commit_details.splitlines():
matcher = re.match(details_matcher, a_commit)
if not matcher:
continue
(summary, date, author_email, author_name) = matcher.groups()
author_email = mailmap.get(author_email, author_email)
try:
date = iso8601.parse_date(date)
except iso8601.ParseError:
date = None
if self._skip_entry(summary, date, author_email, author_name):
continue
log.append({
'summary': translate_utf8(summary),
'when': date,
'author_email': translate_utf8(author_email),
'author_name': translate_utf8(author_name),
})
# Bucketize the dates by day
date_buckets = collections.defaultdict(list)
for entry in log:
day = entry['when'].date()
date_buckets[day].append(entry)
return date_buckets
def format_log(self):
date_buckets = self.get_log()
lines = []
for d in reversed(sorted(date_buckets.keys())):
entries = date_buckets[d]
for entry in entries:
header = "* %s %s <%s>" % (d.strftime("%a %b %d %Y"),
entry['author_name'],
entry['author_email'])
lines.append(header)
summary = entry['summary']
sublines = textwrap.wrap(summary, 77)
if len(sublines):
lines.append("- %s" % sublines[0])
if len(sublines) > 1:
for subline in sublines[1:]:
lines.append(" %s" % subline)
return "\n".join(lines)
def package_name_python2rpm(python_name):
python_name = python_name.lower()
try:
@ -303,12 +147,6 @@ def create_parser():
action="store_true",
default=False,
help="Only generate source RPM")
parser.add_argument(
"--no-changelog",
action="store_true",
default=False,
dest="no_changelog",
help="Do not attempt to build a changelog automatically.")
parser.add_argument(
"--rpm-base",
metavar="<dir>",
@ -421,19 +259,6 @@ def build_epoch_map(options):
epoch_map[name] = epoch
def build_changelog(source_dir, options):
if options.no_changelog:
return ""
cls = None
for c in [GitChangeLog,]:
if c.can_build_for(source_dir):
cls = c
break
if not cls:
return ""
return "\n".join(['', '%changelog', cls(source_dir).format_log(), ''])
def run_egg_info(source_dir, options):
script = """
__file__ = __SETUP_PY__
@ -694,5 +519,5 @@ mv -f INSTALLED_FILES{.tmp,}
if __name__ == "__main__":
try:
main()
except Exception as ex:
print >> sys.stderr, ex
except Exception as exp:
print >> sys.stderr, exp