| #!/usr/bin/env python3 |
| # -*- coding: UTF-8 -*- |
| |
| # Polly/LLVM update_check.py |
| # Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command. |
| |
| import argparse |
| import os |
| import subprocess |
| import shlex |
| import re |
| |
| |
| polly_src_dir = '''@POLLY_SOURCE_DIR@''' |
| polly_lib_dir = '''@POLLY_LIB_DIR@''' |
| shlibext = '''@LLVM_SHLIBEXT@''' |
| llvm_tools_dir = '''@LLVM_TOOLS_DIR@''' |
| llvm_polly_link_into_tools = not '''@LLVM_POLLY_LINK_INTO_TOOLS@'''.lower() in {'','0','n','no','off','false','notfound','llvm_polly_link_into_tools-notfound'} |
| |
| runre = re.compile(r'\s*\;\s*RUN\s*\:(?P<tool>.*)') |
| filecheckre = re.compile(r'\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)') |
| emptyline = re.compile(r'\s*(\;\s*)?') |
| commentline = re.compile(r'\s*(\;.*)?') |
| |
| |
| def ltrim_emptylines(lines,meta=None): |
| while len(lines) and emptyline.fullmatch(lines[0]): |
| del lines[0] |
| if meta is not None: |
| del meta[0] |
| |
| |
| def rtrim_emptylines(lines): |
| while len(lines) and emptyline.fullmatch(lines[-1]): |
| del lines[-1] |
| |
| |
| def trim_emptylines(lines): |
| ltrim_emptylines(lines) |
| rtrim_emptylines(lines) |
| |
| |
| def complete_exename(path, filename): |
| complpath = os.path.join(path, filename) |
| if os.path.isfile(complpath): |
| return complpath |
| elif os.path.isfile(complpath + '.exe'): |
| return complpath + '.exe' |
| return filename |
| |
| |
| def indention(line): |
| for i,c in enumerate(line): |
| if c != ' ' and c != '\t': |
| return i |
| return None |
| |
| |
| def common_indent(lines): |
| indentions = (indention(line) for line in lines) |
| indentions = (indent for indent in indentions if indent is not None) |
| return min(indentions,default=0) |
| |
| |
| funcre = re.compile(r'^ Function: \S*$') |
| regionre = re.compile(r'^ Region: \S*$') |
| depthre = re.compile(r'^ Max Loop Depth: .*') |
| paramre = re.compile(r' [0-9a-z-A-Z_]+\: .*') |
| |
| def classyfier1(lines): |
| i = iter(lines) |
| line = i.__next__() |
| while True: |
| if line.startswith("Printing analysis 'Polly - Calculate dependences' for region: "): |
| yield {'PrintingDependenceInfo'} |
| elif line.startswith("remark: "): |
| yield {'Remark'} |
| elif funcre.fullmatch(line): |
| yield {'Function'} |
| elif regionre.fullmatch(line): |
| yield { 'Region'} |
| elif depthre.fullmatch(line): |
| yield {'MaxLoopDepth'} |
| elif line == ' Invariant Accesses: {': |
| while True: |
| yield { 'InvariantAccesses'} |
| if line == ' }': |
| break |
| line = i.__next__() |
| elif line == ' Context:': |
| yield {'Context'} |
| line = i.__next__() |
| yield {'Context'} |
| elif line == ' Assumed Context:': |
| yield {'AssumedContext'} |
| line = i.__next__() |
| yield {'AssumedContext'} |
| elif line == ' Invalid Context:': |
| yield {'InvalidContext'} |
| line = i.__next__() |
| yield {'InvalidContext'} |
| elif line == ' Boundary Context:': |
| yield {'BoundaryContext'} |
| line = i.__next__() |
| yield {'BoundaryContext'} |
| line = i.__next__() |
| while paramre.fullmatch(line): |
| yield {'Param'} |
| line = i.__next__() |
| continue |
| elif line == ' Arrays {': |
| while True: |
| yield {'Arrays'} |
| if line == ' }': |
| break |
| line = i.__next__() |
| elif line == ' Arrays (Bounds as pw_affs) {': |
| while True: |
| yield {'PwAffArrays'} |
| if line == ' }': |
| break |
| line = i.__next__() |
| elif line.startswith(' Alias Groups ('): |
| while True: |
| yield {'AliasGroups'} |
| line = i.__next__() |
| if not line.startswith(' '): |
| break |
| continue |
| elif line == ' Statements {': |
| while True: |
| yield {'Statements'} |
| if line == ' }': |
| break |
| line = i.__next__() |
| elif line == ' RAW dependences:': |
| yield {'RAWDep','BasicDep','Dep','DepInfo'} |
| line = i.__next__() |
| while line.startswith(' '): |
| yield {'RAWDep','BasicDep','Dep','DepInfo'} |
| line = i.__next__() |
| continue |
| elif line == ' WAR dependences:': |
| yield {'WARDep','BasicDep','Dep','DepInfo'} |
| line = i.__next__() |
| while line.startswith(' '): |
| yield {'WARDep','BasicDep','Dep','DepInfo'} |
| line = i.__next__() |
| continue |
| elif line == ' WAW dependences:': |
| yield {'WAWDep','BasicDep','Dep','DepInfo'} |
| line = i.__next__() |
| while line.startswith(' '): |
| yield {'WAWDep','BasicDep','Dep','DepInfo'} |
| line = i.__next__() |
| continue |
| elif line == ' Reduction dependences:': |
| yield {'RedDep','Dep','DepInfo'} |
| line = i.__next__() |
| while line.startswith(' '): |
| yield {'RedDep','Dep','DepInfo'} |
| line = i.__next__() |
| continue |
| elif line == ' Transitive closure of reduction dependences:': |
| yield {'TransitiveClosureDep','DepInfo'} |
| line = i.__next__() |
| while line.startswith(' '): |
| yield {'TransitiveClosureDep','DepInfo'} |
| line = i.__next__() |
| continue |
| elif line.startswith("New access function '"): |
| yield {'NewAccessFunction'} |
| elif line == 'Schedule before flattening {': |
| while True: |
| yield {'ScheduleBeforeFlattening'} |
| if line == '}': |
| break |
| line = i.__next__() |
| elif line == 'Schedule after flattening {': |
| while True: |
| yield {'ScheduleAfterFlattening'} |
| if line == '}': |
| break |
| line = i.__next__() |
| else: |
| yield set() |
| line = i.__next__() |
| |
| |
| def classyfier2(lines): |
| i = iter(lines) |
| line = i.__next__() |
| while True: |
| if funcre.fullmatch(line): |
| while line.startswith(' '): |
| yield {'FunctionDetail'} |
| line = i.__next__() |
| continue |
| elif line.startswith("Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "): |
| yield {'PrintingIslAst'} |
| line = i.__next__() |
| while not line.startswith('Printing analysis'): |
| yield {'AstDetail'} |
| line = i.__next__() |
| continue |
| else: |
| yield set() |
| line = i.__next__() |
| |
| |
| replrepl = {'{{':'{{[{][{]}}','}}': '{{[}][}]}}', '[[':'{{\[\[}}',']]': '{{\]\]}}'} |
| replre = re.compile('|'.join(re.escape(k) for k in replrepl.keys())) |
| |
| def main(): |
| parser = argparse.ArgumentParser(description="Update CHECK lines") |
| parser.add_argument('testfile',help="File to update (absolute or relative to --testdir)") |
| parser.add_argument('--check-style',choices=['CHECK','CHECK-NEXT'],default='CHECK-NEXT',help="What kind of checks lines to generate") |
| parser.add_argument('--check-position',choices=['end','before-content','autodetect'],default='autodetect',help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there") |
| parser.add_argument('--check-include',action='append',default=[], help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)") |
| parser.add_argument('--check-label-include',action='append',default=[],help="Use CHECK-LABEL for these includes") |
| parser.add_argument('--check-part-newline',action='store_true',help="Add empty line between different check parts") |
| parser.add_argument('--prefix-only',action='append',default=None,help="Update only these prefixes (default: all)") |
| parser.add_argument('--bindir',help="Location of the opt program") |
| parser.add_argument('--testdir',help="Root dir for unit tests") |
| parser.add_argument('--inplace','-i',action='store_true',help="Replace input file") |
| parser.add_argument('--output','-o',help="Write changed input to this file") |
| known = parser.parse_args() |
| |
| if not known.inplace and known.output is None: |
| print("Must specify what to do with output (--output or --inplace)") |
| exit(1) |
| if known.inplace and known.output is not None: |
| print("--inplace and --output are mutually exclusive") |
| exit(1) |
| |
| outfile = known.output |
| |
| filecheckparser = argparse.ArgumentParser(add_help=False) |
| filecheckparser.add_argument('-check-prefix','--check-prefix',default='CHECK') |
| |
| filename = known.testfile |
| for dir in ['.', known.testdir, os.path.join(polly_src_dir,'test'), polly_src_dir]: |
| if not dir: |
| continue |
| testfilename = os.path.join(dir,filename) |
| if os.path.isfile(testfilename): |
| filename = testfilename |
| break |
| |
| if known.inplace: |
| outfile = filename |
| |
| allchecklines = [] |
| checkprefixes = [] |
| |
| with open(filename, 'r') as file: |
| oldlines = [line.rstrip('\r\n') for line in file.readlines()] |
| |
| runlines = [] |
| for line in oldlines: |
| m = runre.match(line) |
| if m: |
| runlines.append(m.group('tool')) |
| |
| continuation = '' |
| newrunlines = [] |
| for line in runlines: |
| if line.endswith('\\'): |
| continuation += line[:-2] + ' ' |
| else: |
| newrunlines.append(continuation + line) |
| continuation = '' |
| if continuation: |
| newrunlines.append(continuation) |
| |
| for line in newrunlines: |
| m = filecheckre.match(line) |
| if not m: |
| continue |
| |
| tool, filecheck = m.group('tool', 'filecheck') |
| filecheck = shlex.split(filecheck) |
| tool = shlex.split(tool) |
| if known.bindir is not None: |
| tool[0] = complete_exename(known.bindir, tool[0]) |
| if os.path.isdir(llvm_tools_dir): |
| tool[0] = complete_exename(llvm_tools_dir, tool[0]) |
| check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix |
| if known.prefix_only is not None and not check_prefix in known.prefix_only: |
| continue |
| if check_prefix in checkprefixes: |
| continue |
| checkprefixes.append(check_prefix) |
| |
| newtool = [] |
| optstderr = None |
| for toolarg in tool: |
| toolarg = toolarg.replace('%s', filename) |
| toolarg = toolarg.replace('%S', os.path.dirname(filename)) |
| if toolarg == '%loadPolly': |
| if not llvm_polly_link_into_tools: |
| newtool += ['-load',os.path.join(polly_lib_dir,'LLVMPolly' + shlibext)] |
| newtool.append('-polly-process-unprofitable') |
| newtool.append('-polly-remarks-minimal') |
| elif toolarg == '2>&1': |
| optstderr = subprocess.STDOUT |
| else: |
| newtool.append(toolarg) |
| tool = newtool |
| |
| inpfile = None |
| i = 1 |
| while i < len(tool): |
| if tool[i] == '<': |
| inpfile = tool[i + 1] |
| del tool[i:i + 2] |
| continue |
| i += 1 |
| if inpfile: |
| with open(inpfile) as inp: |
| retlines = subprocess.check_output(tool,universal_newlines=True,stdin=inp,stderr=optstderr) |
| else: |
| retlines = subprocess.check_output(tool,universal_newlines=True,stderr=optstderr) |
| retlines = [line.replace('\t', ' ') for line in retlines.splitlines()] |
| check_include = [] |
| for checkme in known.check_include + known.check_label_include: |
| parts = checkme.split('=') |
| if len(parts) == 2: |
| if parts[0] == check_prefix: |
| check_include.append(parts[1]) |
| else: |
| check_include.append(checkme) |
| |
| if check_include: |
| filtered_retlines = [] |
| classified_retlines = [] |
| lastmatch = None |
| for line,kind in ((line,class1.union(class2)) for line,class1,class2 in zip(retlines,classyfier1(retlines), classyfier2(retlines))): |
| match = kind.intersection(check_include) |
| if match: |
| if lastmatch != match: |
| filtered_retlines.append('') |
| classified_retlines.append({'Separator'}) |
| filtered_retlines.append(line) |
| classified_retlines.append(kind) |
| lastmatch = match |
| |
| retlines = filtered_retlines |
| else: |
| classified_retlines = (set() for line in retlines) |
| |
| rtrim_emptylines(retlines) |
| ltrim_emptylines(retlines,classified_retlines) |
| retlines = [replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines] |
| indent = common_indent(retlines) |
| retlines = [line[indent:] for line in retlines] |
| checklines = [] |
| previous_was_empty = True |
| for line,kind in zip(retlines,classified_retlines): |
| if line: |
| if known.check_style == 'CHECK' and known.check_label_include: |
| if not kind.isdisjoint(known.check_label_include): |
| checklines.append('; ' + check_prefix + '-LABEL: ' + line) |
| else: |
| checklines.append('; ' + check_prefix + ': ' + line) |
| elif known.check_style == 'CHECK': |
| checklines.append('; ' + check_prefix + ': ' + line) |
| elif known.check_label_include and known.check_label_include: |
| if not kind.isdisjoint(known.check_label_include): |
| checklines.append('; ' + check_prefix + '-LABEL: ' + line) |
| elif previous_was_empty: |
| checklines.append('; ' + check_prefix + ': ' + line) |
| else: |
| checklines.append('; ' + check_prefix + '-NEXT: ' + line) |
| else: |
| if previous_was_empty: |
| checklines.append('; ' + check_prefix + ': ' + line) |
| else: |
| checklines.append('; ' + check_prefix + '-NEXT: ' + line) |
| previous_was_empty = False |
| else: |
| if not 'Separator' in kind or known.check_part_newline: |
| checklines.append(';') |
| previous_was_empty = True |
| allchecklines.append(checklines) |
| |
| if not checkprefixes: |
| return |
| |
| checkre = re.compile(r'^\s*\;\s*(' + '|'.join([re.escape(s) for s in checkprefixes]) + ')(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:') |
| firstcheckline = None |
| firstnoncommentline = None |
| headerlines = [] |
| newlines = [] |
| uptonowlines = [] |
| emptylines = [] |
| lastwascheck = False |
| for line in oldlines: |
| if checkre.match(line): |
| if firstcheckline is None: |
| firstcheckline = len(newlines) + len(emptylines) |
| if not lastwascheck: |
| uptonowlines += emptylines |
| emptylines = [] |
| lastwascheck = True |
| elif emptyline.fullmatch(line): |
| emptylines.append(line) |
| else: |
| newlines += uptonowlines |
| newlines += emptylines |
| newlines.append(line) |
| emptylines = [] |
| uptonowlines = [] |
| lastwascheck = False |
| |
| for i,line in enumerate(newlines): |
| if not commentline.fullmatch(line): |
| firstnoncommentline = i |
| break |
| |
| with open(outfile,'w',newline='') as file: |
| def writelines(lines): |
| for line in lines: |
| file.write(line) |
| file.write('\n') |
| |
| if firstcheckline is not None and known.check_position == 'autodetect': |
| writelines(newlines[:firstcheckline]) |
| writelines(uptonowlines) |
| for i,checklines in enumerate(allchecklines): |
| if i != 0: |
| file.write('\n') |
| writelines(checklines) |
| writelines(newlines[firstcheckline:]) |
| writelines(emptylines) |
| elif firstnoncommentline is not None and known.check_position == 'before-content': |
| headerlines = newlines[:firstnoncommentline] |
| rtrim_emptylines(headerlines) |
| contentlines = newlines[firstnoncommentline:] |
| ltrim_emptylines(contentlines) |
| |
| writelines(headerlines) |
| for checklines in allchecklines: |
| file.write('\n') |
| writelines(checklines) |
| file.write('\n') |
| writelines(contentlines) |
| writelines(uptonowlines) |
| writelines(emptylines) |
| else: |
| writelines(newlines) |
| rtrim_emptylines(newlines) |
| for checklines in allchecklines: |
| file.write('\n\n') |
| writelines(checklines) |
| |
| |
| if __name__ == '__main__': |
| main() |
