utils/benchmark.py

#!/usr/bin/env python
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ==------------------------------------------------------------------------==#

import os
import glob
import re
import subprocess
import json
import datetime
import argparse

try:
    from urllib.parse import urlencode
    from urllib.request import urlopen, Request
except ImportError:
    from urllib import urlencode
    from urllib2 import urlopen, Request


parser = argparse.ArgumentParser()
parser.add_argument("benchmark_directory")
parser.add_argument("--runs", type=int, default=10)
parser.add_argument("--wrapper", default="")
parser.add_argument("--machine", required=True)
parser.add_argument("--revision", required=True)
parser.add_argument("--threads", action="store_true")
parser.add_argument(
    "--url",
    help="The lnt server url to send the results to",
    default="http://localhost:8000/db_default/v4/link/submitRun",
)
args = parser.parse_args()


class Bench:
    def __init__(self, directory, variant):
        self.directory = directory
        self.variant = variant

    def __str__(self):
        if not self.variant:
            return self.directory
        return "%s-%s" % (self.directory, self.variant)


def getBenchmarks():
    ret = []
    for i in glob.glob("*/response*.txt"):
        m = re.match("response-(.*)\.txt", os.path.basename(i))
        variant = m.groups()[0] if m else None
        ret.append(Bench(os.path.dirname(i), variant))
    return ret


def parsePerfNum(num):
    num = num.replace(b",", b"")
    try:
        return int(num)
    except ValueError:
        return float(num)


def parsePerfLine(line):
    ret = {}
    line = line.split(b"#")[0].strip()
    if len(line) != 0:
        p = line.split()
        ret[p[1].strip().decode("ascii")] = parsePerfNum(p[0])
    return ret


def parsePerf(output):
    ret = {}
    lines = [x.strip() for x in output.split(b"\n")]

    seconds = [x for x in lines if b"seconds time elapsed" in x][0]
    seconds = seconds.strip().split()[0].strip()
    ret["seconds-elapsed"] = parsePerfNum(seconds)

    measurement_lines = [x for x in lines if b"#" in x]
    for l in measurement_lines:
        ret.update(parsePerfLine(l))
    return ret


def run(cmd):
    try:
        return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        print(e.output)
        raise e


def combinePerfRun(acc, d):
    for k, v in d.items():
        a = acc.get(k, [])
        a.append(v)
        acc[k] = a


def perf(cmd):
    # Discard the first run to warm up any system cache.
    run(cmd)

    ret = {}
    wrapper_args = [x for x in args.wrapper.split(",") if x]
    for i in range(args.runs):
        os.unlink("t")
        out = run(wrapper_args + ["perf", "stat"] + cmd)
        r = parsePerf(out)
        combinePerfRun(ret, r)
    os.unlink("t")
    return ret


def runBench(bench):
    thread_arg = [] if args.threads else ["--no-threads"]
    os.chdir(bench.directory)
    suffix = "-%s" % bench.variant if bench.variant else ""
    response = "response" + suffix + ".txt"
    ret = perf(["../ld.lld", "@" + response, "-o", "t"] + thread_arg)
    ret["name"] = str(bench)
    os.chdir("..")
    return ret


def buildLntJson(benchmarks):
    start = datetime.datetime.utcnow().isoformat()
    tests = [runBench(b) for b in benchmarks]
    end = datetime.datetime.utcnow().isoformat()
    ret = {
        "format_version": 2,
        "machine": {"name": args.machine},
        "run": {
            "end_time": start,
            "start_time": end,
            "llvm_project_revision": args.revision,
        },
        "tests": tests,
    }
    return json.dumps(ret, sort_keys=True, indent=4)


def submitToServer(data):
    data2 = urlencode({"input_data": data}).encode("ascii")
    urlopen(Request(args.url, data2))


os.chdir(args.benchmark_directory)
data = buildLntJson(getBenchmarks())
submitToServer(data)