| #!/usr/bin/env python3 |
| # -*- coding: UTF-8 -*- |
| |
| # Polly/LLVM update_check.py |
| # Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command. |
| |
| import argparse |
| import os |
| import subprocess |
| import shlex |
| import re |
| |
| |
| polly_src_dir = """@POLLY_SOURCE_DIR@""" |
| polly_lib_dir = """@POLLY_LIB_DIR@""" |
| shlibext = """@LLVM_SHLIBEXT@""" |
| llvm_tools_dir = """@LLVM_TOOLS_DIR@""" |
| llvm_polly_link_into_tools = not """@LLVM_POLLY_LINK_INTO_TOOLS@""".lower() in { |
| "", |
| "0", |
| "n", |
| "no", |
| "off", |
| "false", |
| "notfound", |
| "llvm_polly_link_into_tools-notfound", |
| } |
| |
| runre = re.compile(r"\s*\;\s*RUN\s*\:(?P<tool>.*)") |
| filecheckre = re.compile(r"\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)") |
| emptyline = re.compile(r"\s*(\;\s*)?") |
| commentline = re.compile(r"\s*(\;.*)?") |
| |
| |
| def ltrim_emptylines(lines, meta=None): |
| while len(lines) and emptyline.fullmatch(lines[0]): |
| del lines[0] |
| if meta is not None: |
| del meta[0] |
| |
| |
| def rtrim_emptylines(lines): |
| while len(lines) and emptyline.fullmatch(lines[-1]): |
| del lines[-1] |
| |
| |
| def trim_emptylines(lines): |
| ltrim_emptylines(lines) |
| rtrim_emptylines(lines) |
| |
| |
| def complete_exename(path, filename): |
| complpath = os.path.join(path, filename) |
| if os.path.isfile(complpath): |
| return complpath |
| elif os.path.isfile(complpath + ".exe"): |
| return complpath + ".exe" |
| return filename |
| |
| |
| def indention(line): |
| for i, c in enumerate(line): |
| if c != " " and c != "\t": |
| return i |
| return None |
| |
| |
| def common_indent(lines): |
| indentions = (indention(line) for line in lines) |
| indentions = (indent for indent in indentions if indent is not None) |
| return min(indentions, default=0) |
| |
| |
| funcre = re.compile(r"^ Function: \S*$") |
| regionre = re.compile(r"^ Region: \S*$") |
| depthre = re.compile(r"^ Max Loop Depth: .*") |
| paramre = re.compile(r" [0-9a-z-A-Z_]+\: .*") |
| |
| |
| def classyfier1(lines): |
| i = iter(lines) |
| line = i.__next__() |
| while True: |
| if line.startswith( |
| "Printing analysis 'Polly - Calculate dependences' for region: " |
| ): |
| yield {"PrintingDependenceInfo"} |
| elif line.startswith("remark: "): |
| yield {"Remark"} |
| elif funcre.fullmatch(line): |
| yield {"Function"} |
| elif regionre.fullmatch(line): |
| yield {"Region"} |
| elif depthre.fullmatch(line): |
| yield {"MaxLoopDepth"} |
| elif line == " Invariant Accesses: {": |
| while True: |
| yield {"InvariantAccesses"} |
| if line == " }": |
| break |
| line = i.__next__() |
| elif line == " Context:": |
| yield {"Context"} |
| line = i.__next__() |
| yield {"Context"} |
| elif line == " Assumed Context:": |
| yield {"AssumedContext"} |
| line = i.__next__() |
| yield {"AssumedContext"} |
| elif line == " Invalid Context:": |
| yield {"InvalidContext"} |
| line = i.__next__() |
| yield {"InvalidContext"} |
| elif line == " Boundary Context:": |
| yield {"BoundaryContext"} |
| line = i.__next__() |
| yield {"BoundaryContext"} |
| line = i.__next__() |
| while paramre.fullmatch(line): |
| yield {"Param"} |
| line = i.__next__() |
| continue |
| elif line == " Arrays {": |
| while True: |
| yield {"Arrays"} |
| if line == " }": |
| break |
| line = i.__next__() |
| elif line == " Arrays (Bounds as pw_affs) {": |
| while True: |
| yield {"PwAffArrays"} |
| if line == " }": |
| break |
| line = i.__next__() |
| elif line.startswith(" Alias Groups ("): |
| while True: |
| yield {"AliasGroups"} |
| line = i.__next__() |
| if not line.startswith(" "): |
| break |
| continue |
| elif line == " Statements {": |
| while True: |
| yield {"Statements"} |
| if line == " }": |
| break |
| line = i.__next__() |
| elif line == " RAW dependences:": |
| yield {"RAWDep", "BasicDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| while line.startswith(" "): |
| yield {"RAWDep", "BasicDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| continue |
| elif line == " WAR dependences:": |
| yield {"WARDep", "BasicDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| while line.startswith(" "): |
| yield {"WARDep", "BasicDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| continue |
| elif line == " WAW dependences:": |
| yield {"WAWDep", "BasicDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| while line.startswith(" "): |
| yield {"WAWDep", "BasicDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| continue |
| elif line == " Reduction dependences:": |
| yield {"RedDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| while line.startswith(" "): |
| yield {"RedDep", "Dep", "DepInfo"} |
| line = i.__next__() |
| continue |
| elif line == " Transitive closure of reduction dependences:": |
| yield {"TransitiveClosureDep", "DepInfo"} |
| line = i.__next__() |
| while line.startswith(" "): |
| yield {"TransitiveClosureDep", "DepInfo"} |
| line = i.__next__() |
| continue |
| elif line.startswith("New access function '"): |
| yield {"NewAccessFunction"} |
| elif line == "Schedule before flattening {": |
| while True: |
| yield {"ScheduleBeforeFlattening"} |
| if line == "}": |
| break |
| line = i.__next__() |
| elif line == "Schedule after flattening {": |
| while True: |
| yield {"ScheduleAfterFlattening"} |
| if line == "}": |
| break |
| line = i.__next__() |
| else: |
| yield set() |
| line = i.__next__() |
| |
| |
| def classyfier2(lines): |
| i = iter(lines) |
| line = i.__next__() |
| while True: |
| if funcre.fullmatch(line): |
| while line.startswith(" "): |
| yield {"FunctionDetail"} |
| line = i.__next__() |
| continue |
| elif line.startswith( |
| "Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: " |
| ): |
| yield {"PrintingIslAst"} |
| line = i.__next__() |
| while not line.startswith("Printing analysis"): |
| yield {"AstDetail"} |
| line = i.__next__() |
| continue |
| else: |
| yield set() |
| line = i.__next__() |
| |
| |
| replrepl = {"{{": "{{[{][{]}}", "}}": "{{[}][}]}}", "[[": "{{\[\[}}", "]]": "{{\]\]}}"} |
| replre = re.compile("|".join(re.escape(k) for k in replrepl.keys())) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description="Update CHECK lines") |
| parser.add_argument( |
| "testfile", help="File to update (absolute or relative to --testdir)" |
| ) |
| parser.add_argument( |
| "--check-style", |
| choices=["CHECK", "CHECK-NEXT"], |
| default="CHECK-NEXT", |
| help="What kind of checks lines to generate", |
| ) |
| parser.add_argument( |
| "--check-position", |
| choices=["end", "before-content", "autodetect"], |
| default="autodetect", |
| help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there", |
| ) |
| parser.add_argument( |
| "--check-include", |
| action="append", |
| default=[], |
| help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)", |
| ) |
| parser.add_argument( |
| "--check-label-include", |
| action="append", |
| default=[], |
| help="Use CHECK-LABEL for these includes", |
| ) |
| parser.add_argument( |
| "--check-part-newline", |
| action="store_true", |
| help="Add empty line between different check parts", |
| ) |
| parser.add_argument( |
| "--prefix-only", |
| action="append", |
| default=None, |
| help="Update only these prefixes (default: all)", |
| ) |
| parser.add_argument("--bindir", help="Location of the opt program") |
| parser.add_argument("--testdir", help="Root dir for unit tests") |
| parser.add_argument( |
| "--inplace", "-i", action="store_true", help="Replace input file" |
| ) |
| parser.add_argument("--output", "-o", help="Write changed input to this file") |
| known = parser.parse_args() |
| |
| if not known.inplace and known.output is None: |
| print("Must specify what to do with output (--output or --inplace)") |
| exit(1) |
| if known.inplace and known.output is not None: |
| print("--inplace and --output are mutually exclusive") |
| exit(1) |
| |
| outfile = known.output |
| |
| filecheckparser = argparse.ArgumentParser(add_help=False) |
| filecheckparser.add_argument("-check-prefix", "--check-prefix", default="CHECK") |
| |
| filename = known.testfile |
| for dir in [".", known.testdir, os.path.join(polly_src_dir, "test"), polly_src_dir]: |
| if not dir: |
| continue |
| testfilename = os.path.join(dir, filename) |
| if os.path.isfile(testfilename): |
| filename = testfilename |
| break |
| |
| if known.inplace: |
| outfile = filename |
| |
| allchecklines = [] |
| checkprefixes = [] |
| |
| with open(filename, "r") as file: |
| oldlines = [line.rstrip("\r\n") for line in file.readlines()] |
| |
| runlines = [] |
| for line in oldlines: |
| m = runre.match(line) |
| if m: |
| runlines.append(m.group("tool")) |
| |
| continuation = "" |
| newrunlines = [] |
| for line in runlines: |
| if line.endswith("\\"): |
| continuation += line[:-2] + " " |
| else: |
| newrunlines.append(continuation + line) |
| continuation = "" |
| if continuation: |
| newrunlines.append(continuation) |
| |
| for line in newrunlines: |
| m = filecheckre.match(line) |
| if not m: |
| continue |
| |
| tool, filecheck = m.group("tool", "filecheck") |
| filecheck = shlex.split(filecheck) |
| tool = shlex.split(tool) |
| if known.bindir is not None: |
| tool[0] = complete_exename(known.bindir, tool[0]) |
| if os.path.isdir(llvm_tools_dir): |
| tool[0] = complete_exename(llvm_tools_dir, tool[0]) |
| check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix |
| if known.prefix_only is not None and not check_prefix in known.prefix_only: |
| continue |
| if check_prefix in checkprefixes: |
| continue |
| checkprefixes.append(check_prefix) |
| |
| newtool = [] |
| optstderr = None |
| for toolarg in tool: |
| toolarg = toolarg.replace("%s", filename) |
| toolarg = toolarg.replace("%S", os.path.dirname(filename)) |
| if toolarg == "%loadPolly": |
| if not llvm_polly_link_into_tools: |
| newtool += [ |
| "-load", |
| os.path.join(polly_lib_dir, "LLVMPolly" + shlibext), |
| ] |
| newtool.append("-polly-process-unprofitable") |
| newtool.append("-polly-remarks-minimal") |
| elif toolarg == "2>&1": |
| optstderr = subprocess.STDOUT |
| else: |
| newtool.append(toolarg) |
| tool = newtool |
| |
| inpfile = None |
| i = 1 |
| while i < len(tool): |
| if tool[i] == "<": |
| inpfile = tool[i + 1] |
| del tool[i : i + 2] |
| continue |
| i += 1 |
| if inpfile: |
| with open(inpfile) as inp: |
| retlines = subprocess.check_output( |
| tool, universal_newlines=True, stdin=inp, stderr=optstderr |
| ) |
| else: |
| retlines = subprocess.check_output( |
| tool, universal_newlines=True, stderr=optstderr |
| ) |
| retlines = [line.replace("\t", " ") for line in retlines.splitlines()] |
| check_include = [] |
| for checkme in known.check_include + known.check_label_include: |
| parts = checkme.split("=") |
| if len(parts) == 2: |
| if parts[0] == check_prefix: |
| check_include.append(parts[1]) |
| else: |
| check_include.append(checkme) |
| |
| if check_include: |
| filtered_retlines = [] |
| classified_retlines = [] |
| lastmatch = None |
| for line, kind in ( |
| (line, class1.union(class2)) |
| for line, class1, class2 in zip( |
| retlines, classyfier1(retlines), classyfier2(retlines) |
| ) |
| ): |
| match = kind.intersection(check_include) |
| if match: |
| if lastmatch != match: |
| filtered_retlines.append("") |
| classified_retlines.append({"Separator"}) |
| filtered_retlines.append(line) |
| classified_retlines.append(kind) |
| lastmatch = match |
| |
| retlines = filtered_retlines |
| else: |
| classified_retlines = (set() for line in retlines) |
| |
| rtrim_emptylines(retlines) |
| ltrim_emptylines(retlines, classified_retlines) |
| retlines = [ |
| replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines |
| ] |
| indent = common_indent(retlines) |
| retlines = [line[indent:] for line in retlines] |
| checklines = [] |
| previous_was_empty = True |
| for line, kind in zip(retlines, classified_retlines): |
| if line: |
| if known.check_style == "CHECK" and known.check_label_include: |
| if not kind.isdisjoint(known.check_label_include): |
| checklines.append("; " + check_prefix + "-LABEL: " + line) |
| else: |
| checklines.append("; " + check_prefix + ": " + line) |
| elif known.check_style == "CHECK": |
| checklines.append("; " + check_prefix + ": " + line) |
| elif known.check_label_include and known.check_label_include: |
| if not kind.isdisjoint(known.check_label_include): |
| checklines.append("; " + check_prefix + "-LABEL: " + line) |
| elif previous_was_empty: |
| checklines.append("; " + check_prefix + ": " + line) |
| else: |
| checklines.append("; " + check_prefix + "-NEXT: " + line) |
| else: |
| if previous_was_empty: |
| checklines.append("; " + check_prefix + ": " + line) |
| else: |
| checklines.append("; " + check_prefix + "-NEXT: " + line) |
| previous_was_empty = False |
| else: |
| if not "Separator" in kind or known.check_part_newline: |
| checklines.append(";") |
| previous_was_empty = True |
| allchecklines.append(checklines) |
| |
| if not checkprefixes: |
| return |
| |
| checkre = re.compile( |
| r"^\s*\;\s*(" |
| + "|".join([re.escape(s) for s in checkprefixes]) |
| + ")(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:" |
| ) |
| firstcheckline = None |
| firstnoncommentline = None |
| headerlines = [] |
| newlines = [] |
| uptonowlines = [] |
| emptylines = [] |
| lastwascheck = False |
| for line in oldlines: |
| if checkre.match(line): |
| if firstcheckline is None: |
| firstcheckline = len(newlines) + len(emptylines) |
| if not lastwascheck: |
| uptonowlines += emptylines |
| emptylines = [] |
| lastwascheck = True |
| elif emptyline.fullmatch(line): |
| emptylines.append(line) |
| else: |
| newlines += uptonowlines |
| newlines += emptylines |
| newlines.append(line) |
| emptylines = [] |
| uptonowlines = [] |
| lastwascheck = False |
| |
| for i, line in enumerate(newlines): |
| if not commentline.fullmatch(line): |
| firstnoncommentline = i |
| break |
| |
| with open(outfile, "w", newline="") as file: |
| |
| def writelines(lines): |
| for line in lines: |
| file.write(line) |
| file.write("\n") |
| |
| if firstcheckline is not None and known.check_position == "autodetect": |
| writelines(newlines[:firstcheckline]) |
| writelines(uptonowlines) |
| for i, checklines in enumerate(allchecklines): |
| if i != 0: |
| file.write("\n") |
| writelines(checklines) |
| writelines(newlines[firstcheckline:]) |
| writelines(emptylines) |
| elif ( |
| firstnoncommentline is not None and known.check_position == "before-content" |
| ): |
| headerlines = newlines[:firstnoncommentline] |
| rtrim_emptylines(headerlines) |
| contentlines = newlines[firstnoncommentline:] |
| ltrim_emptylines(contentlines) |
| |
| writelines(headerlines) |
| for checklines in allchecklines: |
| file.write("\n") |
| writelines(checklines) |
| file.write("\n") |
| writelines(contentlines) |
| writelines(uptonowlines) |
| writelines(emptylines) |
| else: |
| writelines(newlines) |
| rtrim_emptylines(newlines) |
| for checklines in allchecklines: |
| file.write("\n\n") |
| writelines(checklines) |
| |
| |
| if __name__ == "__main__": |
| main() |
