test/update_check.py

   1 #! /usr/bin/env python3
   2 # -*- coding: UTF-8 -*-
   3
   4 # Polly/LLVM update_check.py
   5 # Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.
   6
   7 import argparse
   8 import os
   9 import subprocess
  10 import shlex
  11 import re
  12
  13
  14 polly_src_dir = '''@POLLY_SOURCE_DIR@'''
  15 polly_lib_dir = '''@POLLY_LIB_DIR@'''
  16 shlibext = '''@LLVM_SHLIBEXT@'''
  17 llvm_tools_dir = '''@LLVM_TOOLS_DIR@'''
  18 link_polly_into_tools = not '''@LINK_POLLY_INTO_TOOLS@'''.lower() in {'','0','n','no','off','false','notfound','link_polly_into_tools-notfound'}
  19
  20 runre = re.compile(r'\s*\;\s*RUN\s*\:(?P<tool>.*)')
  21 filecheckre = re.compile(r'\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)')
  22 emptyline = re.compile(r'\s*(\;\s*)?')
  23 commentline = re.compile(r'\s*(\;.*)?')
  24
  25
  26 def ltrim_emptylines(lines,meta=None):
  27     while len(lines) and emptyline.fullmatch(lines[0]):
  28         del lines[0]
  29         if meta is not None:
  30             del meta[0]
  31
  32
  33 def rtrim_emptylines(lines):
  34     while len(lines) and emptyline.fullmatch(lines[-1]):
  35         del lines[-1]
  36
  37
  38 def trim_emptylines(lines):
  39     ltrim_emptylines(lines)
  40     rtrim_emptylines(lines)
  41
  42
  43 def complete_exename(path, filename):
  44     complpath = os.path.join(path, filename)
  45     if os.path.isfile(complpath):
  46         return complpath
  47     elif os.path.isfile(complpath + '.exe'):
  48         return complpath + '.exe'
  49     return filename
  50
  51
  52 def indention(line):
  53     for i,c in enumerate(line):
  54         if c != ' ' and c != '\t':
  55             return i
  56     return None
  57
  58
  59 def common_indent(lines):
  60     indentions = (indention(line) for line in lines)
  61     indentions = (indent for indent in indentions if indent is not None)
  62     return min(indentions,default=0)
  63
  64
  65 funcre = re.compile(r'^    Function: \S*$')
  66 regionre = re.compile(r'^    Region: \S*$')
  67 depthre = re.compile(r'^    Max Loop Depth: .*')
  68 paramre = re.compile(r'    [0-9a-z-A-Z_]+\: .*')
  69
  70 def classyfier1(lines):
  71     i = iter(lines)
  72     line = i.__next__()
  73     while True:
  74         if line.startswith("Printing analysis 'Polly - Calculate dependences' for region: "):
  75             yield {'PrintingDependenceInfo'}
  76         elif line.startswith("remark: "):
  77             yield {'Remark'}
  78         elif funcre.fullmatch(line):
  79             yield {'Function'}
  80         elif regionre.fullmatch(line):
  81             yield  { 'Region'}
  82         elif depthre.fullmatch(line):
  83             yield  {'MaxLoopDepth'}
  84         elif line == '    Invariant Accesses: {':
  85             while True:
  86                 yield { 'InvariantAccesses'}
  87                 if line == '    }':
  88                     break
  89                 line = i.__next__()
  90         elif line == '    Context:':
  91             yield  {'Context'}
  92             line = i.__next__()
  93             yield  {'Context'}
  94         elif line == '    Assumed Context:':
  95             yield  {'AssumedContext'}
  96             line = i.__next__()
  97             yield  {'AssumedContext'}
  98         elif line == '    Invalid Context:':
  99             yield  {'InvalidContext'}
 100             line = i.__next__()
 101             yield  {'InvalidContext'}
 102         elif line == '    Boundary Context:':
 103             yield  {'BoundaryContext'}
 104             line = i.__next__()
 105             yield  {'BoundaryContext'}
 106             line = i.__next__()
 107             while paramre.fullmatch(line):
 108                 yield  {'Param'}
 109                 line = i.__next__()
 110             continue
 111         elif line == '    Arrays {':
 112             while True:
 113                 yield  {'Arrays'}
 114                 if line == '    }':
 115                     break
 116                 line = i.__next__()
 117         elif line == '    Arrays (Bounds as pw_affs) {':
 118             while True:
 119                 yield  {'PwAffArrays'}
 120                 if line == '    }':
 121                     break
 122                 line = i.__next__()
 123         elif line.startswith('    Alias Groups ('):
 124             while True:
 125                 yield  {'AliasGroups'}
 126                 line = i.__next__()
 127                 if not line.startswith('        '):
 128                     break
 129             continue
 130         elif line == '    Statements {':
 131             while True:
 132                 yield  {'Statements'}
 133                 if line == '    }':
 134                     break
 135                 line = i.__next__()
 136         elif line == '    RAW dependences:':
 137             yield {'RAWDep','BasicDep','Dep','DepInfo'}
 138             line = i.__next__()
 139             while line.startswith('        '):
 140                 yield  {'RAWDep','BasicDep','Dep','DepInfo'}
 141                 line = i.__next__()
 142             continue
 143         elif line == '    WAR dependences:':
 144             yield {'WARDep','BasicDep','Dep','DepInfo'}
 145             line = i.__next__()
 146             while line.startswith('        '):
 147                 yield  {'WARDep','BasicDep','Dep','DepInfo'}
 148                 line = i.__next__()
 149             continue
 150         elif line == '    WAW dependences:':
 151             yield {'WAWDep','BasicDep','Dep','DepInfo'}
 152             line = i.__next__()
 153             while line.startswith('        '):
 154                 yield  {'WAWDep','BasicDep','Dep','DepInfo'}
 155                 line = i.__next__()
 156             continue
 157         elif line == '    Reduction dependences:':
 158             yield {'RedDep','Dep','DepInfo'}
 159             line = i.__next__()
 160             while line.startswith('        '):
 161                 yield  {'RedDep','Dep','DepInfo'}
 162                 line = i.__next__()
 163             continue
 164         elif line == '    Transitive closure of reduction dependences:':
 165             yield {'TransitiveClosureDep','DepInfo'}
 166             line = i.__next__()
 167             while line.startswith('        '):
 168                 yield  {'TransitiveClosureDep','DepInfo'}
 169                 line = i.__next__()
 170             continue
 171         else:
 172             yield set()
 173         line = i.__next__()
 174
 175
 176 def classyfier2(lines):
 177     i = iter(lines)
 178     line = i.__next__()
 179     while True:
 180         if funcre.fullmatch(line):
 181             while line.startswith('    '):
 182                 yield  {'FunctionDetail'}
 183                 line = i.__next__()
 184             continue
 185         elif line.startswith("Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "):
 186             yield {'PrintingIslAst'}
 187             line = i.__next__()
 188             while not line.startswith('Printing analysis'):
 189                 yield  {'AstDetail'}
 190                 line = i.__next__()
 191             continue
 192         else:
 193             yield set()
 194         line = i.__next__()
 195
 196
 197 replrepl = {'{{':'{{[{][{]}}','}}': '{{[}][}]}}', '[[':'{{\[\[}}',']]': '{{\]\]}}'}
 198 replre = re.compile('|'.join(re.escape(k) for k in replrepl.keys()))
 199
 200 def main():
 201     parser = argparse.ArgumentParser(description="Update CHECK lines")
 202     parser.add_argument('testfile',help="File to update (absolute or relative to --testdir)")
 203     parser.add_argument('--check-style',choices=['CHECK','CHECK-NEXT'],default='CHECK-NEXT',help="What kind of checks lines to generate")
 204     parser.add_argument('--check-position',choices=['end','before-content','autodetect'],default='autodetect',help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there")
 205     parser.add_argument('--check-include',action='append',default=[], help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)")
 206     parser.add_argument('--check-label-include',action='append',default=[],help="Use CHECK-LABEL for these includes")
 207     parser.add_argument('--check-part-newline',action='store_true',help="Add empty line between different check parts")
 208     parser.add_argument('--prefix-only',action='append',default=None,help="Update only these prefixes (default: all)")
 209     parser.add_argument('--bindir',help="Location of the opt program")
 210     parser.add_argument('--testdir',help="Root dir for unit tests")
 211     parser.add_argument('--inplace','-i',action='store_true',help="Replace input file")
 212     parser.add_argument('--output','-o',help="Write changed input to this file")
 213     known = parser.parse_args()
 214
 215     if not known.inplace and known.output is None:
 216         print("Must specify what to do with output (--output or --inplace)")
 217         exit(1)
 218     if known.inplace and known.output is not None:
 219         print("--inplace and --output are mutually exclusive")
 220         exit(1)
 221
 222     outfile = known.output
 223
 224     filecheckparser = argparse.ArgumentParser(add_help=False)
 225     filecheckparser.add_argument('-check-prefix','--check-prefix',default='CHECK')
 226
 227     filename = known.testfile
 228     for dir in ['.', known.testdir, os.path.join(polly_src_dir,'test'), polly_src_dir]:
 229         if not dir:
 230             continue
 231         testfilename = os.path.join(dir,filename)
 232         if os.path.isfile(testfilename):
 233             filename = testfilename
 234             break
 235
 236     if known.inplace:
 237         outfile = filename
 238
 239     allchecklines = []
 240     checkprefixes = []
 241
 242     with open(filename, 'r') as file:
 243         oldlines = [line.rstrip('\r\n') for line in file.readlines()]
 244
 245     runlines = []
 246     for line in oldlines:
 247         m = runre.match(line)
 248         if m:
 249             runlines.append(m.group('tool'))
 250
 251     continuation = ''
 252     newrunlines = []
 253     for line in runlines:
 254         if line.endswith('\\'):
 255             continuation += line[:-2] + ' '
 256         else:
 257             newrunlines.append(continuation + line)
 258             continuation = ''
 259     if continuation:
 260         newrunlines.append(continuation)
 261
 262     for line in newrunlines:
 263         m = filecheckre.match(line)
 264         if not m:
 265             continue
 266
 267         tool, filecheck = m.group('tool', 'filecheck')
 268         filecheck = shlex.split(filecheck)
 269         tool = shlex.split(tool)
 270         if known.bindir is not None:
 271             tool[0] = complete_exename(known.bindir, tool[0])
 272         if os.path.isdir(llvm_tools_dir):
 273             tool[0] = complete_exename(llvm_tools_dir, tool[0])
 274         check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix
 275         if known.prefix_only is not None and not check_prefix in known.prefix_only:
 276             continue
 277         if check_prefix in checkprefixes:
 278             continue
 279         checkprefixes.append(check_prefix)
 280
 281         newtool = []
 282         optstderr = None
 283         for toolarg in tool:
 284             toolarg = toolarg.replace('%s', filename)
 285             toolarg = toolarg.replace('%S', os.path.dirname(filename))
 286             if toolarg == '%loadPolly':
 287                 if not link_polly_into_tools:
 288                     newtool += ['-load',os.path.join(polly_lib_dir,'LLVMPolly' + shlibext)]
 289                 newtool.append('-polly-process-unprofitable')
 290                 newtool.append('-polly-remarks-minimal')
 291             elif toolarg == '2>&1':
 292                 optstderr = subprocess.STDOUT
 293             else:
 294                 newtool.append(toolarg)
 295         tool = newtool
 296
 297         inpfile = None
 298         i = 1
 299         while i <  len(tool):
 300             if tool[i] == '<':
 301                 inpfile = tool[i + 1]
 302                 del tool[i:i + 2]
 303                 continue
 304             i += 1
 305         if inpfile:
 306             with open(inpfile) as inp:
 307                 retlines = subprocess.check_output(tool,universal_newlines=True,stdin=inp,stderr=optstderr)
 308         else:
 309             retlines = subprocess.check_output(tool,universal_newlines=True,stderr=optstderr)
 310         retlines = [line.replace('\t', '    ') for line in retlines.splitlines()]
 311         check_include = []
 312         for checkme in known.check_include + known.check_label_include:
 313             parts = checkme.split('=')
 314             if len(parts) == 2:
 315                 if parts[0] == check_prefix:
 316                     check_include.append(parts[1])
 317             else:
 318                 check_include.append(checkme)
 319
 320         if check_include:
 321             filtered_retlines = []
 322             classified_retlines = []
 323             lastmatch = None
 324             for line,kind in ((line,class1.union(class2)) for line,class1,class2 in zip(retlines,classyfier1(retlines), classyfier2(retlines))):
 325                 match = kind.intersection(check_include)
 326                 if match:
 327                     if lastmatch != match:
 328                         filtered_retlines.append('')
 329                         classified_retlines.append({'Separator'})
 330                     filtered_retlines.append(line)
 331                     classified_retlines.append(kind)
 332                 lastmatch = match
 333
 334             retlines = filtered_retlines
 335         else:
 336             classified_retlines = (set() for line in retlines)
 337
 338         rtrim_emptylines(retlines)
 339         ltrim_emptylines(retlines,classified_retlines)
 340         retlines = [replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines]
 341         indent = common_indent(retlines)
 342         retlines = [line[indent:] for line in retlines]
 343         checklines = []
 344         previous_was_empty = True
 345         for line,kind in zip(retlines,classified_retlines):
 346             if line:
 347                 if known.check_style == 'CHECK' and known.check_label_include:
 348                     if not kind.isdisjoint(known.check_label_include):
 349                         checklines.append('; ' + check_prefix + '-LABEL: ' + line)
 350                     else:
 351                         checklines.append('; ' + check_prefix + ':       ' + line)
 352                 elif known.check_style == 'CHECK':
 353                     checklines.append('; ' + check_prefix + ': ' + line)
 354                 elif known.check_label_include and known.check_label_include:
 355                     if not kind.isdisjoint(known.check_label_include):
 356                         checklines.append('; ' + check_prefix + '-LABEL: ' + line)
 357                     elif previous_was_empty:
 358                         checklines.append('; ' + check_prefix + ':       ' + line)
 359                     else:
 360                         checklines.append('; ' + check_prefix + '-NEXT:  ' + line)
 361                 else:
 362                     if previous_was_empty:
 363                         checklines.append('; ' + check_prefix + ':      ' + line)
 364                     else:
 365                         checklines.append('; ' + check_prefix + '-NEXT: ' + line)
 366                 previous_was_empty = False
 367             else:
 368                 if not 'Separator' in kind or known.check_part_newline:
 369                     checklines.append(';')
 370                 previous_was_empty = True
 371         allchecklines.append(checklines)
 372
 373     if not checkprefixes:
 374         return
 375
 376     checkre = re.compile(r'^\s*\;\s*(' + '|'.join([re.escape(s) for s in checkprefixes]) + ')(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:')
 377     firstcheckline = None
 378     firstnoncommentline = None
 379     headerlines = []
 380     newlines = []
 381     uptonowlines = []
 382     emptylines = []
 383     lastwascheck = False
 384     for line in oldlines:
 385         if checkre.match(line):
 386             if firstcheckline is None:
 387                 firstcheckline = len(newlines) + len(emptylines)
 388             if not lastwascheck:
 389                 uptonowlines += emptylines
 390             emptylines = []
 391             lastwascheck = True
 392         elif emptyline.fullmatch(line):
 393             emptylines.append(line)
 394         else:
 395             newlines += uptonowlines
 396             newlines += emptylines
 397             newlines.append(line)
 398             emptylines = []
 399             uptonowlines = []
 400             lastwascheck = False
 401
 402     for i,line in enumerate(newlines):
 403         if not commentline.fullmatch(line):
 404             firstnoncommentline = i
 405             break
 406
 407     with open(outfile,'w',newline='') as file:
 408         def writelines(lines):
 409             for line in lines:
 410                 file.write(line)
 411                 file.write('\n')
 412
 413         if firstcheckline is not None and known.check_position == 'autodetect':
 414             writelines(newlines[:firstcheckline])
 415             writelines(uptonowlines)
 416             for i,checklines in enumerate(allchecklines):
 417                 if i != 0:
 418                     file.write('\n')
 419                 writelines(checklines)
 420             writelines(newlines[firstcheckline:])
 421             writelines(emptylines)
 422         elif firstnoncommentline is not None and known.check_position == 'before-content':
 423             headerlines = newlines[:firstnoncommentline]
 424             rtrim_emptylines(headerlines)
 425             contentlines = newlines[firstnoncommentline:]
 426             ltrim_emptylines(contentlines)
 427
 428             writelines(headerlines)
 429             for checklines in allchecklines:
 430                 file.write('\n')
 431                 writelines(checklines)
 432             file.write('\n')
 433             writelines(contentlines)
 434             writelines(uptonowlines)
 435             writelines(emptylines)
 436         else:
 437             writelines(newlines)
 438             rtrim_emptylines(newlines)
 439             for checklines in allchecklines:
 440                 file.write('\n\n')
 441                 writelines(checklines)
 442
 443
 444 if __name__ == '__main__':
 445     main()