contrib/mklog.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) 2020 Free Software Foundation, Inc.
   4 #
   5 # This file is part of GCC.
   6 #
   7 # GCC is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3, or (at your option)
  10 # any later version.
  11 #
  12 # GCC is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with GCC; see the file COPYING.  If not, write to
  19 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
  20 # Boston, MA 02110-1301, USA.
  21
  22 # This script parses a .diff file generated with 'diff -up' or 'diff -cp'
  23 # and adds a skeleton ChangeLog file to the file. It does not try to be
  24 # too smart when parsing function names, but it produces a reasonable
  25 # approximation.
  26 #
  27 # Author: Martin Liska <mliska@suse.cz>
  28
  29 import argparse
  30 import datetime
  31 import os
  32 import re
  33 import subprocess
  34 import sys
  35 from itertools import takewhile
  36
  37 import requests
  38
  39 from unidiff import PatchSet
  40
  41 pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)')
  42 dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)')
  43 dg_regex = re.compile(r'{\s+dg-(error|warning)')
  44 identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)')
  45 comment_regex = re.compile(r'^\/\*')
  46 struct_regex = re.compile(r'^(class|struct|union|enum)\s+'
  47                           r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)')
  48 macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)')
  49 super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)')
  50 fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]')
  51 template_and_param_regex = re.compile(r'<[^<>]*>')
  52 md_def_regex = re.compile(r'\(define.*\s+"(.*)"')
  53 bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \
  54                'include_fields=summary'
  55
  56 function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'}
  57
  58 help_message = """\
  59 Generate ChangeLog template for PATCH.
  60 PATCH must be generated using diff(1)'s -up or -cp options
  61 (or their equivalent in git).
  62 """
  63
  64 script_folder = os.path.realpath(__file__)
  65 gcc_root = os.path.dirname(os.path.dirname(script_folder))
  66
  67
  68 def find_changelog(path):
  69     folder = os.path.split(path)[0]
  70     while True:
  71         if os.path.exists(os.path.join(gcc_root, folder, 'ChangeLog')):
  72             return folder
  73         folder = os.path.dirname(folder)
  74         if folder == '':
  75             return folder
  76     raise AssertionError()
  77
  78
  79 def extract_function_name(line):
  80     if comment_regex.match(line):
  81         return None
  82     m = struct_regex.search(line)
  83     if m:
  84         # Struct declaration
  85         return m.group(1) + ' ' + m.group(3)
  86     m = macro_regex.search(line)
  87     if m:
  88         # Macro definition
  89         return m.group(2)
  90     m = super_macro_regex.search(line)
  91     if m:
  92         # Supermacro
  93         return m.group(1)
  94     m = fn_regex.search(line)
  95     if m:
  96         # Discard template and function parameters.
  97         fn = m.group(1)
  98         fn = re.sub(template_and_param_regex, '', fn)
  99         return fn.rstrip()
 100     return None
 101
 102
 103 def try_add_function(functions, line):
 104     fn = extract_function_name(line)
 105     if fn and fn not in functions:
 106         functions.append(fn)
 107     return bool(fn)
 108
 109
 110 def sort_changelog_files(changed_file):
 111     return (changed_file.is_added_file, changed_file.is_removed_file)
 112
 113
 114 def get_pr_titles(prs):
 115     output = ''
 116     for pr in prs:
 117         pr_id = pr.split('/')[-1]
 118         r = requests.get(bugzilla_url % pr_id)
 119         bugs = r.json()['bugs']
 120         if len(bugs) == 1:
 121             output += '%s - %s\n' % (pr, bugs[0]['summary'])
 122             print(output)
 123     if output:
 124         output += '\n'
 125     return output
 126
 127
 128 def generate_changelog(data, no_functions=False, fill_pr_titles=False):
 129     changelogs = {}
 130     changelog_list = []
 131     prs = []
 132     out = ''
 133     diff = PatchSet(data)
 134
 135     for file in diff:
 136         # skip files that can't be parsed
 137         if file.path == '/dev/null':
 138             continue
 139         changelog = find_changelog(file.path)
 140         if changelog not in changelogs:
 141             changelogs[changelog] = []
 142             changelog_list.append(changelog)
 143         changelogs[changelog].append(file)
 144
 145         # Extract PR entries from newly added tests
 146         if 'testsuite' in file.path and file.is_added_file:
 147             # Only search first ten lines as later lines may
 148             # contains commented code which a note that it
 149             # has not been tested due to a certain PR or DR.
 150             for line in list(file)[0][0:10]:
 151                 m = pr_regex.search(line.value)
 152                 if m:
 153                     pr = m.group('pr')
 154                     if pr not in prs:
 155                         prs.append(pr)
 156                 else:
 157                     m = dr_regex.search(line.value)
 158                     if m:
 159                         dr = m.group('dr')
 160                         if dr not in prs:
 161                             prs.append(dr)
 162                     elif dg_regex.search(line.value):
 163                         # Found dg-warning/dg-error line
 164                         break
 165
 166     if fill_pr_titles:
 167         out += get_pr_titles(prs)
 168
 169     # sort ChangeLog so that 'testsuite' is at the end
 170     for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x):
 171         files = changelogs[changelog]
 172         out += '%s:\n' % os.path.join(changelog, 'ChangeLog')
 173         out += '\n'
 174         for pr in prs:
 175             out += '\t%s\n' % pr
 176         # new and deleted files should be at the end
 177         for file in sorted(files, key=sort_changelog_files):
 178             assert file.path.startswith(changelog)
 179             in_tests = 'testsuite' in changelog or 'testsuite' in file.path
 180             relative_path = file.path[len(changelog):].lstrip('/')
 181             functions = []
 182             if file.is_added_file:
 183                 msg = 'New test' if in_tests else 'New file'
 184                 out += '\t* %s: %s.\n' % (relative_path, msg)
 185             elif file.is_removed_file:
 186                 out += '\t* %s: Removed.\n' % (relative_path)
 187             elif hasattr(file, 'is_rename') and file.is_rename:
 188                 out += '\t* %s: Moved to...\n' % (relative_path)
 189                 new_path = file.target_file[2:]
 190                 # A file can be theoretically moved to a location that
 191                 # belongs to a different ChangeLog.  Let user fix it.
 192                 if new_path.startswith(changelog):
 193                     new_path = new_path[len(changelog):].lstrip('/')
 194                 out += '\t* %s: ...here.\n' % (new_path)
 195             else:
 196                 if not no_functions:
 197                     for hunk in file:
 198                         # Do not add function names for testsuite files
 199                         extension = os.path.splitext(relative_path)[1]
 200                         if not in_tests and extension in function_extensions:
 201                             last_fn = None
 202                             modified_visited = False
 203                             success = False
 204                             for line in hunk:
 205                                 m = identifier_regex.match(line.value)
 206                                 if line.is_added or line.is_removed:
 207                                     # special-case definition in .md files
 208                                     m2 = md_def_regex.match(line.value)
 209                                     if extension == '.md' and m2:
 210                                         fn = m2.group(1)
 211                                         if fn not in functions:
 212                                             functions.append(fn)
 213                                             last_fn = None
 214                                             success = True
 215
 216                                     if not line.value.strip():
 217                                         continue
 218                                     modified_visited = True
 219                                     if m and try_add_function(functions,
 220                                                               m.group(1)):
 221                                         last_fn = None
 222                                         success = True
 223                                 elif line.is_context:
 224                                     if last_fn and modified_visited:
 225                                         try_add_function(functions, last_fn)
 226                                         last_fn = None
 227                                         modified_visited = False
 228                                         success = True
 229                                     elif m:
 230                                         last_fn = m.group(1)
 231                                         modified_visited = False
 232                             if not success:
 233                                 try_add_function(functions,
 234                                                  hunk.section_header)
 235                 if functions:
 236                     out += '\t* %s (%s):\n' % (relative_path, functions[0])
 237                     for fn in functions[1:]:
 238                         out += '\t(%s):\n' % fn
 239                 else:
 240                     out += '\t* %s:\n' % relative_path
 241         out += '\n'
 242     return out
 243
 244
 245 def update_copyright(data):
 246     current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d')
 247     username = subprocess.check_output('git config user.name', shell=True,
 248                                        encoding='utf8').strip()
 249     email = subprocess.check_output('git config user.email', shell=True,
 250                                     encoding='utf8').strip()
 251
 252     changelogs = set()
 253     diff = PatchSet(data)
 254
 255     for file in diff:
 256         changelog = os.path.join(find_changelog(file.path), 'ChangeLog')
 257         if changelog not in changelogs:
 258             changelogs.add(changelog)
 259             with open(changelog) as f:
 260                 content = f.read()
 261             with open(changelog, 'w+') as f:
 262                 f.write(f'{current_timestamp}  {username}  <{email}>\n\n')
 263                 f.write('\tUpdate copyright years.\n\n')
 264                 f.write(content)
 265
 266
 267 if __name__ == '__main__':
 268     parser = argparse.ArgumentParser(description=help_message)
 269     parser.add_argument('input', nargs='?',
 270                         help='Patch file (or missing, read standard input)')
 271     parser.add_argument('-s', '--no-functions', action='store_true',
 272                         help='Do not generate function names in ChangeLogs')
 273     parser.add_argument('-p', '--fill-up-bug-titles', action='store_true',
 274                         help='Download title of mentioned PRs')
 275     parser.add_argument('-c', '--changelog',
 276                         help='Append the ChangeLog to a git commit message '
 277                              'file')
 278     parser.add_argument('--update-copyright', action='store_true',
 279                         help='Update copyright in ChangeLog files')
 280     args = parser.parse_args()
 281     if args.input == '-':
 282         args.input = None
 283
 284     data = open(args.input) if args.input else sys.stdin
 285     if args.update_copyright:
 286         update_copyright(data)
 287     else:
 288         output = generate_changelog(data, args.no_functions,
 289                                     args.fill_up_bug_titles)
 290         if args.changelog:
 291             lines = open(args.changelog).read().split('\n')
 292             start = list(takewhile(lambda l: not l.startswith('#'), lines))
 293             end = lines[len(start):]
 294             with open(args.changelog, 'w') as f:
 295                 if start:
 296                     # appent empty line
 297                     if start[-1] != '':
 298                         start.append('')
 299                 else:
 300                     # append 2 empty lines
 301                     start = 2 * ['']
 302                 f.write('\n'.join(start))
 303                 f.write('\n')
 304                 f.write(output)
 305                 f.write('\n'.join(end))
 306         else:
 307             print(output, end='')