test/update_check.py

   1 #! /usr/bin/env python3
   2 # -*- coding: UTF-8 -*-
   3
   4 # Polly/LLVM update_check.py
   5 # Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.
   6
   7 import argparse
   8 import os
   9 import subprocess
  10 import shlex
  11 import re
  12
  13
  14 polly_src_dir = '''@POLLY_SOURCE_DIR@'''
  15 polly_lib_dir = '''@POLLY_LIB_DIR@'''
  16 shlibext = '''@LLVM_SHLIBEXT@'''
  17 llvm_tools_dir = '''@LLVM_TOOLS_DIR@'''
  18 link_polly_into_tools = not '''@LINK_POLLY_INTO_TOOLS@'''.lower() in {'','0','n','no','off','false','notfound','link_polly_into_tools-notfound'}
  19
  20 runre = re.compile(r'\s*\;\s*RUN\s*\:(?P<tool>.*)')
  21 filecheckre = re.compile(r'\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)')
  22 emptyline = re.compile(r'\s*(\;\s*)?')
  23 commentline = re.compile(r'\s*(\;.*)?')
  24
  25
  26 def ltrim_emptylines(lines,meta=None):
  27     while len(lines) and emptyline.fullmatch(lines[0]):
  28         del lines[0]
  29         if meta is not None:
  30             del meta[0]
  31
  32
  33 def rtrim_emptylines(lines):
  34     while len(lines) and emptyline.fullmatch(lines[-1]):
  35         del lines[-1]
  36
  37
  38 def trim_emptylines(lines):
  39     ltrim_emptylines(lines)
  40     rtrim_emptylines(lines)
  41
  42
  43 def complete_exename(path, filename):
  44     complpath = os.path.join(path, filename)
  45     if os.path.isfile(complpath):
  46         return complpath
  47     elif os.path.isfile(complpath + '.exe'):
  48         return complpath + '.exe'
  49     return filename
  50
  51
  52 def indention(line):
  53     for i,c in enumerate(line):
  54         if c != ' ' and c != '\t':
  55             return i
  56     return None
  57
  58
  59 def common_indent(lines):
  60     indentions = (indention(line) for line in lines)
  61     indentions = (indent for indent in indentions if indent is not None)
  62     return min(indentions,default=0)
  63
  64
  65 funcre = re.compile(r'^    Function: \S*$')
  66 regionre = re.compile(r'^    Region: \S*$')
  67 depthre = re.compile(r'^    Max Loop Depth: .*')
  68 paramre = re.compile(r'    [0-9a-z-A-Z_]+\: .*')
  69
  70 def classyfier1(lines):
  71     i = iter(lines)
  72     line = i.__next__()
  73     while True:
  74         if line.startswith("Printing analysis 'Polly - Calculate dependences' for region: "):
  75             yield {'PrintingDependenceInfo'}
  76         elif line.startswith("remark: "):
  77             yield {'Remark'}
  78         elif funcre.fullmatch(line):
  79             yield {'Function'}
  80         elif regionre.fullmatch(line):
  81             yield  { 'Region'}
  82         elif depthre.fullmatch(line):
  83             yield  {'MaxLoopDepth'}
  84         elif line == '    Invariant Accesses: {':
  85             while True:
  86                 yield { 'InvariantAccesses'}
  87                 if line == '    }':
  88                     break
  89                 line = i.__next__()
  90         elif line == '    Context:':
  91             yield  {'Context'}
  92             line = i.__next__()
  93             yield  {'Context'}
  94         elif line == '    Assumed Context:':
  95             yield  {'AssumedContext'}
  96             line = i.__next__()
  97             yield  {'AssumedContext'}
  98         elif line == '    Invalid Context:':
  99             yield  {'InvalidContext'}
 100             line = i.__next__()
 101             yield  {'InvalidContext'}
 102         elif line == '    Boundary Context:':
 103             yield  {'BoundaryContext'}
 104             line = i.__next__()
 105             yield  {'BoundaryContext'}
 106             line = i.__next__()
 107             while paramre.fullmatch(line):
 108                 yield  {'Param'}
 109                 line = i.__next__()
 110             continue
 111         elif line == '    Arrays {':
 112             while True:
 113                 yield  {'Arrays'}
 114                 if line == '    }':
 115                     break
 116                 line = i.__next__()
 117         elif line == '    Arrays (Bounds as pw_affs) {':
 118             while True:
 119                 yield  {'PwAffArrays'}
 120                 if line == '    }':
 121                     break
 122                 line = i.__next__()
 123         elif line.startswith('    Alias Groups ('):
 124             while True:
 125                 yield  {'AliasGroups'}
 126                 line = i.__next__()
 127                 if not line.startswith('        '):
 128                     break
 129             continue
 130         elif line == '    Statements {':
 131             while True:
 132                 yield  {'Statements'}
 133                 if line == '    }':
 134                     break
 135                 line = i.__next__()
 136         elif line == '    RAW dependences:':
 137             yield {'RAWDep','BasicDep','Dep','DepInfo'}
 138             line = i.__next__()
 139             while line.startswith('        '):
 140                 yield  {'RAWDep','BasicDep','Dep','DepInfo'}
 141                 line = i.__next__()
 142             continue
 143         elif line == '    WAR dependences:':
 144             yield {'WARDep','BasicDep','Dep','DepInfo'}
 145             line = i.__next__()
 146             while line.startswith('        '):
 147                 yield  {'WARDep','BasicDep','Dep','DepInfo'}
 148                 line = i.__next__()
 149             continue
 150         elif line == '    WAW dependences:':
 151             yield {'WAWDep','BasicDep','Dep','DepInfo'}
 152             line = i.__next__()
 153             while line.startswith('        '):
 154                 yield  {'WAWDep','BasicDep','Dep','DepInfo'}
 155                 line = i.__next__()
 156             continue
 157         elif line == '    Reduction dependences:':
 158             yield {'RedDep','Dep','DepInfo'}
 159             line = i.__next__()
 160             while line.startswith('        '):
 161                 yield  {'RedDep','Dep','DepInfo'}
 162                 line = i.__next__()
 163             continue
 164         elif line == '    Transitive closure of reduction dependences:':
 165             yield {'TransitiveClosureDep','DepInfo'}
 166             line = i.__next__()
 167             while line.startswith('        '):
 168                 yield  {'TransitiveClosureDep','DepInfo'}
 169                 line = i.__next__()
 170             continue
 171         elif line.startswith("New access function '"):
 172             yield {'NewAccessFunction'}
 173         elif line == 'Schedule before flattening {':
 174             while True:
 175                 yield  {'ScheduleBeforeFlattening'}
 176                 if line == '}':
 177                     break
 178                 line = i.__next__()
 179         elif line == 'Schedule after flattening {':
 180             while True:
 181                 yield  {'ScheduleAfterFlattening'}
 182                 if line == '}':
 183                     break
 184                 line = i.__next__()
 185         else:
 186             yield set()
 187         line = i.__next__()
 188
 189
 190 def classyfier2(lines):
 191     i = iter(lines)
 192     line = i.__next__()
 193     while True:
 194         if funcre.fullmatch(line):
 195             while line.startswith('    '):
 196                 yield  {'FunctionDetail'}
 197                 line = i.__next__()
 198             continue
 199         elif line.startswith("Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "):
 200             yield {'PrintingIslAst'}
 201             line = i.__next__()
 202             while not line.startswith('Printing analysis'):
 203                 yield  {'AstDetail'}
 204                 line = i.__next__()
 205             continue
 206         else:
 207             yield set()
 208         line = i.__next__()
 209
 210
 211 replrepl = {'{{':'{{[{][{]}}','}}': '{{[}][}]}}', '[[':'{{\[\[}}',']]': '{{\]\]}}'}
 212 replre = re.compile('|'.join(re.escape(k) for k in replrepl.keys()))
 213
 214 def main():
 215     parser = argparse.ArgumentParser(description="Update CHECK lines")
 216     parser.add_argument('testfile',help="File to update (absolute or relative to --testdir)")
 217     parser.add_argument('--check-style',choices=['CHECK','CHECK-NEXT'],default='CHECK-NEXT',help="What kind of checks lines to generate")
 218     parser.add_argument('--check-position',choices=['end','before-content','autodetect'],default='autodetect',help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there")
 219     parser.add_argument('--check-include',action='append',default=[], help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)")
 220     parser.add_argument('--check-label-include',action='append',default=[],help="Use CHECK-LABEL for these includes")
 221     parser.add_argument('--check-part-newline',action='store_true',help="Add empty line between different check parts")
 222     parser.add_argument('--prefix-only',action='append',default=None,help="Update only these prefixes (default: all)")
 223     parser.add_argument('--bindir',help="Location of the opt program")
 224     parser.add_argument('--testdir',help="Root dir for unit tests")
 225     parser.add_argument('--inplace','-i',action='store_true',help="Replace input file")
 226     parser.add_argument('--output','-o',help="Write changed input to this file")
 227     known = parser.parse_args()
 228
 229     if not known.inplace and known.output is None:
 230         print("Must specify what to do with output (--output or --inplace)")
 231         exit(1)
 232     if known.inplace and known.output is not None:
 233         print("--inplace and --output are mutually exclusive")
 234         exit(1)
 235
 236     outfile = known.output
 237
 238     filecheckparser = argparse.ArgumentParser(add_help=False)
 239     filecheckparser.add_argument('-check-prefix','--check-prefix',default='CHECK')
 240
 241     filename = known.testfile
 242     for dir in ['.', known.testdir, os.path.join(polly_src_dir,'test'), polly_src_dir]:
 243         if not dir:
 244             continue
 245         testfilename = os.path.join(dir,filename)
 246         if os.path.isfile(testfilename):
 247             filename = testfilename
 248             break
 249
 250     if known.inplace:
 251         outfile = filename
 252
 253     allchecklines = []
 254     checkprefixes = []
 255
 256     with open(filename, 'r') as file:
 257         oldlines = [line.rstrip('\r\n') for line in file.readlines()]
 258
 259     runlines = []
 260     for line in oldlines:
 261         m = runre.match(line)
 262         if m:
 263             runlines.append(m.group('tool'))
 264
 265     continuation = ''
 266     newrunlines = []
 267     for line in runlines:
 268         if line.endswith('\\'):
 269             continuation += line[:-2] + ' '
 270         else:
 271             newrunlines.append(continuation + line)
 272             continuation = ''
 273     if continuation:
 274         newrunlines.append(continuation)
 275
 276     for line in newrunlines:
 277         m = filecheckre.match(line)
 278         if not m:
 279             continue
 280
 281         tool, filecheck = m.group('tool', 'filecheck')
 282         filecheck = shlex.split(filecheck)
 283         tool = shlex.split(tool)
 284         if known.bindir is not None:
 285             tool[0] = complete_exename(known.bindir, tool[0])
 286         if os.path.isdir(llvm_tools_dir):
 287             tool[0] = complete_exename(llvm_tools_dir, tool[0])
 288         check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix
 289         if known.prefix_only is not None and not check_prefix in known.prefix_only:
 290             continue
 291         if check_prefix in checkprefixes:
 292             continue
 293         checkprefixes.append(check_prefix)
 294
 295         newtool = []
 296         optstderr = None
 297         for toolarg in tool:
 298             toolarg = toolarg.replace('%s', filename)
 299             toolarg = toolarg.replace('%S', os.path.dirname(filename))
 300             if toolarg == '%loadPolly':
 301                 if not link_polly_into_tools:
 302                     newtool += ['-load',os.path.join(polly_lib_dir,'LLVMPolly' + shlibext)]
 303                 newtool.append('-polly-process-unprofitable')
 304                 newtool.append('-polly-remarks-minimal')
 305             elif toolarg == '2>&1':
 306                 optstderr = subprocess.STDOUT
 307             else:
 308                 newtool.append(toolarg)
 309         tool = newtool
 310
 311         inpfile = None
 312         i = 1
 313         while i <  len(tool):
 314             if tool[i] == '<':
 315                 inpfile = tool[i + 1]
 316                 del tool[i:i + 2]
 317                 continue
 318             i += 1
 319         if inpfile:
 320             with open(inpfile) as inp:
 321                 retlines = subprocess.check_output(tool,universal_newlines=True,stdin=inp,stderr=optstderr)
 322         else:
 323             retlines = subprocess.check_output(tool,universal_newlines=True,stderr=optstderr)
 324         retlines = [line.replace('\t', '    ') for line in retlines.splitlines()]
 325         check_include = []
 326         for checkme in known.check_include + known.check_label_include:
 327             parts = checkme.split('=')
 328             if len(parts) == 2:
 329                 if parts[0] == check_prefix:
 330                     check_include.append(parts[1])
 331             else:
 332                 check_include.append(checkme)
 333
 334         if check_include:
 335             filtered_retlines = []
 336             classified_retlines = []
 337             lastmatch = None
 338             for line,kind in ((line,class1.union(class2)) for line,class1,class2 in zip(retlines,classyfier1(retlines), classyfier2(retlines))):
 339                 match = kind.intersection(check_include)
 340                 if match:
 341                     if lastmatch != match:
 342                         filtered_retlines.append('')
 343                         classified_retlines.append({'Separator'})
 344                     filtered_retlines.append(line)
 345                     classified_retlines.append(kind)
 346                 lastmatch = match
 347
 348             retlines = filtered_retlines
 349         else:
 350             classified_retlines = (set() for line in retlines)
 351
 352         rtrim_emptylines(retlines)
 353         ltrim_emptylines(retlines,classified_retlines)
 354         retlines = [replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines]
 355         indent = common_indent(retlines)
 356         retlines = [line[indent:] for line in retlines]
 357         checklines = []
 358         previous_was_empty = True
 359         for line,kind in zip(retlines,classified_retlines):
 360             if line:
 361                 if known.check_style == 'CHECK' and known.check_label_include:
 362                     if not kind.isdisjoint(known.check_label_include):
 363                         checklines.append('; ' + check_prefix + '-LABEL: ' + line)
 364                     else:
 365                         checklines.append('; ' + check_prefix + ':       ' + line)
 366                 elif known.check_style == 'CHECK':
 367                     checklines.append('; ' + check_prefix + ': ' + line)
 368                 elif known.check_label_include and known.check_label_include:
 369                     if not kind.isdisjoint(known.check_label_include):
 370                         checklines.append('; ' + check_prefix + '-LABEL: ' + line)
 371                     elif previous_was_empty:
 372                         checklines.append('; ' + check_prefix + ':       ' + line)
 373                     else:
 374                         checklines.append('; ' + check_prefix + '-NEXT:  ' + line)
 375                 else:
 376                     if previous_was_empty:
 377                         checklines.append('; ' + check_prefix + ':      ' + line)
 378                     else:
 379                         checklines.append('; ' + check_prefix + '-NEXT: ' + line)
 380                 previous_was_empty = False
 381             else:
 382                 if not 'Separator' in kind or known.check_part_newline:
 383                     checklines.append(';')
 384                 previous_was_empty = True
 385         allchecklines.append(checklines)
 386
 387     if not checkprefixes:
 388         return
 389
 390     checkre = re.compile(r'^\s*\;\s*(' + '|'.join([re.escape(s) for s in checkprefixes]) + ')(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:')
 391     firstcheckline = None
 392     firstnoncommentline = None
 393     headerlines = []
 394     newlines = []
 395     uptonowlines = []
 396     emptylines = []
 397     lastwascheck = False
 398     for line in oldlines:
 399         if checkre.match(line):
 400             if firstcheckline is None:
 401                 firstcheckline = len(newlines) + len(emptylines)
 402             if not lastwascheck:
 403                 uptonowlines += emptylines
 404             emptylines = []
 405             lastwascheck = True
 406         elif emptyline.fullmatch(line):
 407             emptylines.append(line)
 408         else:
 409             newlines += uptonowlines
 410             newlines += emptylines
 411             newlines.append(line)
 412             emptylines = []
 413             uptonowlines = []
 414             lastwascheck = False
 415
 416     for i,line in enumerate(newlines):
 417         if not commentline.fullmatch(line):
 418             firstnoncommentline = i
 419             break
 420
 421     with open(outfile,'w',newline='') as file:
 422         def writelines(lines):
 423             for line in lines:
 424                 file.write(line)
 425                 file.write('\n')
 426
 427         if firstcheckline is not None and known.check_position == 'autodetect':
 428             writelines(newlines[:firstcheckline])
 429             writelines(uptonowlines)
 430             for i,checklines in enumerate(allchecklines):
 431                 if i != 0:
 432                     file.write('\n')
 433                 writelines(checklines)
 434             writelines(newlines[firstcheckline:])
 435             writelines(emptylines)
 436         elif firstnoncommentline is not None and known.check_position == 'before-content':
 437             headerlines = newlines[:firstnoncommentline]
 438             rtrim_emptylines(headerlines)
 439             contentlines = newlines[firstnoncommentline:]
 440             ltrim_emptylines(contentlines)
 441
 442             writelines(headerlines)
 443             for checklines in allchecklines:
 444                 file.write('\n')
 445                 writelines(checklines)
 446             file.write('\n')
 447             writelines(contentlines)
 448             writelines(uptonowlines)
 449             writelines(emptylines)
 450         else:
 451             writelines(newlines)
 452             rtrim_emptylines(newlines)
 453             for checklines in allchecklines:
 454                 file.write('\n\n')
 455                 writelines(checklines)
 456
 457
 458 if __name__ == '__main__':
 459     main()