contrib/mklog.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) 2020 Free Software Foundation, Inc.
   4 #
   5 # This file is part of GCC.
   6 #
   7 # GCC is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3, or (at your option)
  10 # any later version.
  11 #
  12 # GCC is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with GCC; see the file COPYING.  If not, write to
  19 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
  20 # Boston, MA 02110-1301, USA.
  21
  22 # This script parses a .diff file generated with 'diff -up' or 'diff -cp'
  23 # and adds a skeleton ChangeLog file to the file. It does not try to be
  24 # too smart when parsing function names, but it produces a reasonable
  25 # approximation.
  26 #
  27 # Author: Martin Liska <mliska@suse.cz>
  28
  29 import argparse
  30 import datetime
  31 import json
  32 import os
  33 import re
  34 import subprocess
  35 import sys
  36 from itertools import takewhile
  37
  38 import requests
  39
  40 from unidiff import PatchSet
  41
  42 LINE_LIMIT = 100
  43 TAB_WIDTH = 8
  44 CO_AUTHORED_BY_PREFIX = 'co-authored-by: '
  45
  46 pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)')
  47 prnum_regex = re.compile(r'PR (?P<comp>[a-z+-]+)/(?P<num>[0-9]+)')
  48 dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)')
  49 dg_regex = re.compile(r'{\s+dg-(error|warning)')
  50 pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P<pr>\d{4,})')
  51 identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)')
  52 comment_regex = re.compile(r'^\/\*')
  53 struct_regex = re.compile(r'^(class|struct|union|enum)\s+'
  54                           r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)')
  55 macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)')
  56 super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)')
  57 fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]')
  58 template_and_param_regex = re.compile(r'<[^<>]*>')
  59 md_def_regex = re.compile(r'\(define.*\s+"(.*)"')
  60 bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \
  61                'include_fields=summary,component'
  62
  63 function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'}
  64
  65 # NB: Makefile.in isn't listed as it's not always generated.
  66 generated_files = {'aclocal.m4', 'config.h.in', 'configure'}
  67
  68 help_message = """\
  69 Generate ChangeLog template for PATCH.
  70 PATCH must be generated using diff(1)'s -up or -cp options
  71 (or their equivalent in git).
  72 """
  73
  74 script_folder = os.path.realpath(__file__)
  75 root = os.path.dirname(os.path.dirname(script_folder))
  76
  77
  78 def find_changelog(path):
  79     folder = os.path.split(path)[0]
  80     while True:
  81         if os.path.exists(os.path.join(root, folder, 'ChangeLog')):
  82             return folder
  83         folder = os.path.dirname(folder)
  84         if folder == '':
  85             return folder
  86     raise AssertionError()
  87
  88
  89 def extract_function_name(line):
  90     if comment_regex.match(line):
  91         return None
  92     m = struct_regex.search(line)
  93     if m:
  94         # Struct declaration
  95         return m.group(1) + ' ' + m.group(3)
  96     m = macro_regex.search(line)
  97     if m:
  98         # Macro definition
  99         return m.group(2)
 100     m = super_macro_regex.search(line)
 101     if m:
 102         # Supermacro
 103         return m.group(1)
 104     m = fn_regex.search(line)
 105     if m:
 106         # Discard template and function parameters.
 107         fn = m.group(1)
 108         fn = re.sub(template_and_param_regex, '', fn)
 109         return fn.rstrip()
 110     return None
 111
 112
 113 def try_add_function(functions, line):
 114     fn = extract_function_name(line)
 115     if fn and fn not in functions:
 116         functions.append(fn)
 117     return bool(fn)
 118
 119
 120 def sort_changelog_files(changed_file):
 121     return (changed_file.is_added_file, changed_file.is_removed_file)
 122
 123
 124 def get_pr_titles(prs):
 125     output = []
 126     for idx, pr in enumerate(prs):
 127         pr_id = pr.split('/')[-1]
 128         r = requests.get(bugzilla_url % pr_id)
 129         bugs = r.json()['bugs']
 130         if len(bugs) == 1:
 131             prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id)
 132             out = '%s - %s\n' % (prs[idx], bugs[0]['summary'])
 133             if out not in output:
 134                 output.append(out)
 135     if output:
 136         output.append('')
 137     return '\n'.join(output)
 138
 139
 140 def append_changelog_line(out, relative_path, text):
 141     line = f'\t* {relative_path}:'
 142     if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT:
 143         out += f'{line} {text}\n'
 144     else:
 145         out += f'{line}\n'
 146         out += f'\t{text}\n'
 147     return out
 148
 149
 150 def get_rel_path_if_prefixed(path, folder):
 151     if path.startswith(folder):
 152         return path[len(folder):].lstrip('/')
 153     else:
 154         return path
 155
 156
 157 def generate_changelog(data, no_functions=False, fill_pr_titles=False,
 158                        additional_prs=None):
 159     global prs
 160     prs = []
 161
 162     changelogs = {}
 163     changelog_list = []
 164     out = ''
 165     diff = PatchSet(data)
 166
 167     if additional_prs:
 168         for apr in additional_prs:
 169             if not apr.startswith('PR ') and '/' in apr:
 170                 apr = 'PR ' + apr
 171             if apr not in prs:
 172                 prs.append(apr)
 173     for file in diff:
 174         # skip files that can't be parsed
 175         if file.path == '/dev/null':
 176             continue
 177         changelog = find_changelog(file.path)
 178         if changelog not in changelogs:
 179             changelogs[changelog] = []
 180             changelog_list.append(changelog)
 181         changelogs[changelog].append(file)
 182
 183         # Extract PR entries from newly added tests
 184         if 'testsuite' in file.path and file.is_added_file:
 185             # Only search first ten lines as later lines may
 186             # contains commented code which a note that it
 187             # has not been tested due to a certain PR or DR.
 188             this_file_prs = []
 189             for line in list(file)[0][0:10]:
 190                 m = pr_regex.search(line.value)
 191                 if m:
 192                     pr = m.group('pr')
 193                     if pr not in prs:
 194                         prs.append(pr)
 195                         this_file_prs.append(pr.split('/')[-1])
 196                 else:
 197                     m = dr_regex.search(line.value)
 198                     if m:
 199                         dr = m.group('dr')
 200                         if dr not in prs:
 201                             prs.append(dr)
 202                             this_file_prs.append(dr.split('/')[-1])
 203                     elif dg_regex.search(line.value):
 204                         # Found dg-warning/dg-error line
 205                         break
 206             # PR number in the file name
 207             fname = os.path.basename(file.path)
 208             m = pr_filename_regex.search(fname)
 209             if m:
 210                 pr = m.group('pr')
 211                 pr2 = 'PR ' + pr
 212                 if pr not in this_file_prs and pr2 not in prs:
 213                     prs.append(pr2)
 214
 215     if fill_pr_titles:
 216         out += get_pr_titles(prs)
 217
 218     # print list of PR entries before ChangeLog entries
 219     if prs:
 220         if not out:
 221             out += '\n'
 222         for pr in prs:
 223             out += '\t%s\n' % pr
 224         out += '\n'
 225
 226     # sort ChangeLog so that 'testsuite' is at the end
 227     for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x):
 228         files = changelogs[changelog]
 229         out += '%s:\n' % os.path.join(changelog, 'ChangeLog')
 230         out += '\n'
 231         # new and deleted files should be at the end
 232         for file in sorted(files, key=sort_changelog_files):
 233             assert file.path.startswith(changelog)
 234             in_tests = 'testsuite' in changelog or 'testsuite' in file.path
 235             relative_path = get_rel_path_if_prefixed(file.path, changelog)
 236             functions = []
 237             if file.is_added_file:
 238                 msg = 'New test.' if in_tests else 'New file.'
 239                 out = append_changelog_line(out, relative_path, msg)
 240             elif file.is_removed_file:
 241                 out = append_changelog_line(out, relative_path, 'Removed.')
 242             elif hasattr(file, 'is_rename') and file.is_rename:
 243                 # A file can be theoretically moved to a location that
 244                 # belongs to a different ChangeLog.  Let user fix it.
 245                 #
 246                 # Since unidiff 0.7.0, path.file == path.target_file[2:],
 247                 # it used to be path.source_file[2:]
 248                 relative_path = get_rel_path_if_prefixed(file.source_file[2:],
 249                                                          changelog)
 250                 out = append_changelog_line(out, relative_path, 'Moved to...')
 251                 new_path = get_rel_path_if_prefixed(file.target_file[2:],
 252                                                     changelog)
 253                 out += f'\t* {new_path}: ...here.\n'
 254             elif os.path.basename(file.path) in generated_files:
 255                 out += '\t* %s: Regenerate.\n' % (relative_path)
 256                 append_changelog_line(out, relative_path, 'Regenerate.')
 257             else:
 258                 if not no_functions:
 259                     for hunk in file:
 260                         # Do not add function names for testsuite files
 261                         extension = os.path.splitext(relative_path)[1]
 262                         if not in_tests and extension in function_extensions:
 263                             last_fn = None
 264                             modified_visited = False
 265                             success = False
 266                             for line in hunk:
 267                                 m = identifier_regex.match(line.value)
 268                                 if line.is_added or line.is_removed:
 269                                     # special-case definition in .md files
 270                                     m2 = md_def_regex.match(line.value)
 271                                     if extension == '.md' and m2:
 272                                         fn = m2.group(1)
 273                                         if fn not in functions:
 274                                             functions.append(fn)
 275                                             last_fn = None
 276                                             success = True
 277
 278                                     if not line.value.strip():
 279                                         continue
 280                                     modified_visited = True
 281                                     if m and try_add_function(functions,
 282                                                               m.group(1)):
 283                                         last_fn = None
 284                                         success = True
 285                                 elif line.is_context:
 286                                     if last_fn and modified_visited:
 287                                         try_add_function(functions, last_fn)
 288                                         last_fn = None
 289                                         modified_visited = False
 290                                         success = True
 291                                     elif m:
 292                                         last_fn = m.group(1)
 293                                         modified_visited = False
 294                             if not success:
 295                                 try_add_function(functions,
 296                                                  hunk.section_header)
 297                 if functions:
 298                     out += '\t* %s (%s):\n' % (relative_path, functions[0])
 299                     for fn in functions[1:]:
 300                         out += '\t(%s):\n' % fn
 301                 else:
 302                     out += '\t* %s:\n' % relative_path
 303         out += '\n'
 304     return out
 305
 306
 307 def update_copyright(data):
 308     current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d')
 309     username = subprocess.check_output('git config user.name', shell=True,
 310                                        encoding='utf8').strip()
 311     email = subprocess.check_output('git config user.email', shell=True,
 312                                     encoding='utf8').strip()
 313
 314     changelogs = set()
 315     diff = PatchSet(data)
 316
 317     for file in diff:
 318         changelog = os.path.join(find_changelog(file.path), 'ChangeLog')
 319         if changelog not in changelogs:
 320             changelogs.add(changelog)
 321             with open(changelog) as f:
 322                 content = f.read()
 323             with open(changelog, 'w+') as f:
 324                 f.write(f'{current_timestamp}  {username}  <{email}>\n\n')
 325                 f.write('\tUpdate copyright years.\n\n')
 326                 f.write(content)
 327
 328
 329 def skip_line_in_changelog(line):
 330     if line.lower().startswith(CO_AUTHORED_BY_PREFIX) or line.startswith('#'):
 331         return False
 332     return True
 333
 334
 335 if __name__ == '__main__':
 336     extra_args = os.getenv('GCC_MKLOG_ARGS')
 337     if extra_args:
 338         sys.argv += json.loads(extra_args)
 339
 340     parser = argparse.ArgumentParser(description=help_message)
 341     parser.add_argument('input', nargs='?',
 342                         help='Patch file (or missing, read standard input)')
 343     parser.add_argument('-b', '--pr-numbers', action='store',
 344                         type=lambda arg: arg.split(','), nargs='?',
 345                         help='Add the specified PRs (comma separated)')
 346     parser.add_argument('-s', '--no-functions', action='store_true',
 347                         help='Do not generate function names in ChangeLogs')
 348     parser.add_argument('-p', '--fill-up-bug-titles', action='store_true',
 349                         help='Download title of mentioned PRs')
 350     parser.add_argument('-d', '--directory',
 351                         help='Root directory where to search for ChangeLog '
 352                         'files')
 353     parser.add_argument('-c', '--changelog',
 354                         help='Append the ChangeLog to a git commit message '
 355                              'file')
 356     parser.add_argument('--update-copyright', action='store_true',
 357                         help='Update copyright in ChangeLog files')
 358     args = parser.parse_args()
 359     if args.input == '-':
 360         args.input = None
 361     if args.directory:
 362         root = args.directory
 363
 364     data = open(args.input) if args.input else sys.stdin
 365     if args.update_copyright:
 366         update_copyright(data)
 367     else:
 368         output = generate_changelog(data, args.no_functions,
 369                                     args.fill_up_bug_titles, args.pr_numbers)
 370         if args.changelog:
 371             lines = open(args.changelog).read().split('\n')
 372             start = list(takewhile(skip_line_in_changelog, lines))
 373             end = lines[len(start):]
 374             with open(args.changelog, 'w') as f:
 375                 if not start or not start[0]:
 376                     if len(prs) == 1:
 377                         # initial commit subject line 'component: [PRnnnnn]'
 378                         m = prnum_regex.match(prs[0])
 379                         if m:
 380                             title = f'{m.group("comp")}: [PR{m.group("num")}]'
 381                             start.insert(0, title)
 382                 if start:
 383                     # append empty line
 384                     if start[-1] != '':
 385                         start.append('')
 386                 else:
 387                     # append 2 empty lines
 388                     start = 2 * ['']
 389                 f.write('\n'.join(start))
 390                 f.write('\n')
 391                 f.write(output)
 392                 f.write('\n'.join(end))
 393         else:
 394             print(output, end='')