scripts/commits-to-changelog.py

   1 #!/usr/bin/python
   2
   3 from optparse import OptionParser
   4 import subprocess
   5 import re
   6 import os.path
   7 import fnmatch
   8 import os
   9 import sys
  10
  11 # subtract 8 for the leading tabstop
  12 fill_column = 74 - 8
  13
  14 path_to_root = None
  15
  16 all_changelogs = {}
  17
  18 def git (command, *args):
  19     popen = subprocess.Popen (["git", command] + list (args), stdout = subprocess.PIPE)
  20     output = popen.communicate () [0]
  21     if popen.returncode != 0:
  22         print >> sys.stderr, "Error: git failed"
  23         exit (1)
  24     return output
  25
  26 def changelog_path (changelog):
  27     global path_to_root
  28     if not path_to_root:
  29         path_to_root = git ("rev-parse", "--show-cdup").strip ()
  30     (pathname, filename) = changelog
  31     return path_to_root + "./" + pathname + "/" + filename
  32
  33 def changelog_for_file (filename):
  34     while filename != "":
  35         dirname = os.path.dirname (filename)
  36         if dirname in all_changelogs:
  37             return (dirname, all_changelogs [dirname])
  38         filename = dirname
  39     assert False
  40
  41 def changelogs_for_file_pattern (pattern, changed_files):
  42     changelogs = set ()
  43     for filename in changed_files:
  44         suffix = filename
  45         while suffix != "":
  46             # FIXME: fnmatch doesn't support the {x,y} pattern
  47             if fnmatch.fnmatch (suffix, pattern):
  48                 changelogs.add (changelog_for_file (filename))
  49             (_, _, suffix) = suffix.partition ("/")
  50     return changelogs
  51
  52 def format_paragraph (paragraph):
  53     lines = []
  54     words = paragraph.split ()
  55     if len (words) == 0:
  56         return lines
  57     current = words [0]
  58     for word in words [1:]:
  59         if len (current) + 1 + len (word) <= fill_column:
  60             current += " " + word
  61         else:
  62             lines.append ("\t" + current)
  63             current = word
  64     lines.append ("\t" + current)
  65     return lines
  66
  67 def format_changelog_paragraph (files, paragraph):
  68     files_string = ""
  69     for (filename, entity) in files:
  70         if len (files_string) > 0:
  71             files_string += ", "
  72         files_string += filename
  73         if entity:
  74             files_string += " (" + entity + ")"
  75     return format_paragraph ("* " + files_string + ": " + paragraph)
  76
  77 def append_paragraph (lines, paragraph):
  78     if len (lines):
  79         lines.append ("")
  80     lines += paragraph
  81
  82 def format_changelog_entries (commit, changed_files, prefix, file_entries, all_paragraphs):
  83     changelogs = set ()
  84     for f in changed_files:
  85         changelogs.add (changelog_for_file (f))
  86     marked_changelogs = set ()
  87
  88     author_line = git ("log", "-n1", "--date=short", "--format=%ad  %an  <%ae>", commit).strip ()
  89
  90     paragraphs = {}
  91     for changelog in changelogs:
  92         paragraphs [changelog] = [author_line]
  93
  94     for (files, comments) in file_entries:
  95         changelog_entries = {}
  96         for (filename, entity) in files:
  97             entry_changelogs = changelogs_for_file_pattern (filename, changed_files)
  98             if len (entry_changelogs) == 0:
  99                 print "Warning: could not match file %s in commit %s" % (filename, commit)
 100             for changelog in entry_changelogs:
 101                 if changelog not in changelog_entries:
 102                     changelog_entries [changelog] = []
 103                 changelog_entries [changelog].append ((filename, entity))
 104                 marked_changelogs.add (changelog)
 105
 106         for (changelog, files) in changelog_entries.items ():
 107             append_paragraph (paragraphs [changelog], format_changelog_paragraph (files, comments [0]))
 108             for paragraph in comments [1:]:
 109                 append_paragraph (paragraphs [changelog], format_paragraph (paragraph))
 110
 111     unmarked_changelogs = changelogs - marked_changelogs
 112     for changelog in unmarked_changelogs:
 113         if len (prefix) == 0:
 114             print "Warning: empty entry in %s for commit %s" % (changelog_path (changelog), commit)
 115             insert_paragraphs = all_paragraphs
 116         else:
 117             insert_paragraphs = prefix
 118         for paragraph in insert_paragraphs:
 119             append_paragraph (paragraphs [changelog], format_paragraph (paragraph))
 120
 121     return paragraphs
 122
 123 def debug_print_commit (commit, raw_message, prefix, file_entries, changed_files, changelog_entries):
 124     print "===================== Commit"
 125     print commit
 126     print "--------------------- RAW"
 127     print raw_message
 128     print "--------------------- Prefix"
 129     for line in prefix:
 130         print line
 131     print "--------------------- File entries"
 132     for (files, comments) in file_entries:
 133         files_str = ""
 134         for (filename, entity) in files:
 135             if len (files_str):
 136                 files_str = files_str + ", "
 137             files_str = files_str + filename
 138             if entity:
 139                 files_str = files_str + " (" + entity + ")"
 140         print files_str
 141         for line in comments:
 142             print "  " + line
 143     print "--------------------- Files touched"
 144     for f in changed_files:
 145         print f
 146     print "--------------------- ChangeLog entries"
 147     for ((dirname, filename), lines) in changelog_entries.items ():
 148         print "%s/%s:" % (dirname, filename)
 149         for line in lines:
 150             print line
 151
 152 def process_commit (commit):
 153     changed_files = map (lambda l: l.split () [2], git ("diff-tree", "--numstat", commit).splitlines () [1:])
 154     if len (filter (lambda f: re.search ("(^|/)Change[Ll]og$", f), changed_files)):
 155         return None
 156     raw_message = git ("log", "-n1", "--format=%B", commit)
 157     # filter SVN migration message
 158     message = re.sub ("(^|\n)svn path=[^\n]+revision=\d+(?=$|\n)", "", raw_message)
 159     # filter ChangeLog headers
 160     message = re.sub ("(^|\n)\d+-\d+-\d+[ \t]+((\w|[.-])+[ \t]+)+<[^\n>]+>(?=$|\n)", "", message)
 161     # filter leading whitespace
 162     message = re.sub ("^\s+", "", message)
 163     # filter trailing whitespace
 164     message = re.sub ("\s+$", "", message)
 165     # paragraphize - first remove whitespace at beginnings and ends of lines
 166     message = re.sub ("[ \t]*\n[ \t]*", "\n", message)
 167     # paragraphize - now replace three or more consecutive newlines with two
 168     message = re.sub ("\n\n\n+", "\n\n", message)
 169     # paragraphize - replace single newlines with a space
 170     message = re.sub ("(?<!\n)\n(?!\n)", " ", message)
 171     # paragraphize - finally, replace double newlines with single ones
 172     message = re.sub ("\n\n", "\n", message)
 173
 174     # A list of paragraphs (strings without newlines) that occur
 175     # before the first file comments
 176     prefix = []
 177
 178     # A list of tuples of the form ([(filename, entity), ...], [paragraph, ...]).
 179     #
 180     # Each describes a file comment, containing multiple paragraphs.
 181     # Those paragraphs belong to a list of files, each with an
 182     # optional entity (usually a function name).
 183     file_entries = []
 184
 185     current_files = None
 186     current_files_comments = None
 187
 188     message_lines = message.splitlines ()
 189     for line in message_lines:
 190         if re.match ("\*\s[^:]+:", line):
 191             if current_files:
 192                 file_entries.append ((current_files, current_files_comments))
 193
 194             (files, _, comments) = line.partition (":")
 195
 196             current_files_comments = [comments.strip ()]
 197
 198             current_files = []
 199             for f in re.split ("\s*,\s*", files [1:].strip ()):
 200                 m = re.search ("\(([^()]+)\)$", f)
 201                 if m:
 202                     filename = f [:m.start (0)].strip ()
 203                     entity = m.group (1).strip ()
 204                 else:
 205                     filename = f
 206                     entity = None
 207                 current_files.append ((filename, entity))
 208         else:
 209             if current_files:
 210                 current_files_comments.append (line)
 211             else:
 212                 prefix.append (line)
 213     if current_files:
 214         file_entries.append ((current_files, current_files_comments))
 215
 216     changelog_entries = format_changelog_entries (commit, changed_files, prefix, file_entries, message_lines)
 217
 218     #debug_print_commit (commit, raw_message, prefix, file_entries, changed_files, changelog_entries)
 219
 220     return changelog_entries
 221
 222 def start_changelog (changelog):
 223     full_path = changelog_path (changelog)
 224     old_name = full_path + ".old"
 225     os.rename (full_path, old_name)
 226     return open (full_path, "w")
 227
 228 def finish_changelog (changelog, file):
 229     old_file = open (changelog_path (changelog) + ".old")
 230     file.write (old_file.read ())
 231     old_file.close ()
 232     file.close ()
 233
 234 def append_lines (file, lines):
 235     for line in lines:
 236         file.write (line + "\n")
 237     file.write ("\n")
 238
 239 def main ():
 240     usage = "usage: %prog [options] <start-commit>"
 241     parser = OptionParser (usage)
 242     parser.add_option ("-r", "--root", dest = "root", help = "Root directory of the working tree to be changed")
 243     (options, args) = parser.parse_args ()
 244     if len (args) != 1:
 245         parser.error ("incorrect number of arguments")
 246     start_commit = args [0]
 247
 248     if options.root:
 249         global path_to_root
 250         path_to_root = options.root + "/"
 251
 252     # MonkeyWrench uses a shared git repo but sets BUILD_REVISION,
 253     # if present we use it instead of HEAD
 254     HEAD = "HEAD"
 255     if 'BUILD_REVISION' in os.environ:
 256         HEAD = os.environ['BUILD_REVISION']
 257
 258     #see if git supports %B in --format
 259     output = git ("log", "-n1", "--format=%B", HEAD)
 260     if output.startswith ("%B"):
 261         print >> sys.stderr, "Error: git doesn't support %B in --format - install version 1.7.2 or newer"
 262         exit (1)
 263
 264     for filename in git ("ls-tree", "-r", "--name-only", HEAD).splitlines ():
 265         if re.search ("(^|/)Change[Ll]og$", filename):
 266             (path, name) = os.path.split (filename)
 267             all_changelogs [path] = name
 268
 269     commits = git ("rev-list", "--no-merges", HEAD, "^%s" % start_commit).splitlines ()
 270
 271     touched_changelogs = {}
 272     for commit in commits:
 273         entries = process_commit (commit)
 274         if entries == None:
 275             continue
 276         for (changelog, lines) in entries.items ():
 277             if not os.path.exists (changelog_path (changelog)):
 278                 continue
 279             if changelog not in touched_changelogs:
 280                 touched_changelogs [changelog] = start_changelog (changelog)
 281             append_lines (touched_changelogs [changelog], lines)
 282     for (changelog, file) in touched_changelogs.items ():
 283         finish_changelog (changelog, file)
 284
 285 if __name__ == "__main__":
 286     main ()