libcxx/utils/compare-benchmarks

#!/usr/bin/env python3

import argparse
import pathlib
import re
import statistics
import sys
import tempfile

import plotly
import tabulate

def parse_lnt(lines):
    """
    Parse lines in LNT format and return a dictionnary of the form:

        {
            'benchmark1': {
                'metric1': [float],
                'metric2': [float],
                ...
            },
            'benchmark2': {
                'metric1': [float],
                'metric2': [float],
                ...
            },
            ...
        }

    Each metric may have multiple values.
    """
    results = {}
    for line in lines:
        line = line.strip()
        if not line:
            continue

        (identifier, value) = line.split(' ')
        (name, metric) = identifier.split('.')
        if name not in results:
            results[name] = {}
        if metric not in results[name]:
            results[name][metric] = []
        results[name][metric].append(float(value))
    return results

def plain_text_comparison(benchmarks, baseline, candidate, baseline_name=None, candidate_name=None):
    """
    Create a tabulated comparison of the baseline and the candidate.
    """
    headers = ['Benchmark', baseline_name, candidate_name, 'Difference', '% Difference']
    fmt = (None, '.2f', '.2f', '.2f', '.2f')
    table = []
    for (bm, base, cand) in zip(benchmarks, baseline, candidate):
        diff = (cand - base) if base and cand else None
        percent = 100 * (diff / base) if base and cand else None
        row = [bm, base, cand, diff, percent]
        table.append(row)
    return tabulate.tabulate(table, headers=headers, floatfmt=fmt, numalign='right')

def create_chart(benchmarks, baseline, candidate, subtitle=None, baseline_name=None, candidate_name=None):
    """
    Create a bar chart comparing 'baseline' and 'candidate'.
    """
    figure = plotly.graph_objects.Figure(layout={
        'title': {
            'text': f'{baseline_name} vs {candidate_name}',
            'subtitle': {'text': subtitle}
        }
    })
    figure.add_trace(plotly.graph_objects.Bar(x=benchmarks, y=baseline, name=baseline_name))
    figure.add_trace(plotly.graph_objects.Bar(x=benchmarks, y=candidate, name=candidate_name))
    return figure

def prepare_series(baseline, candidate, metric, aggregate=statistics.median):
    """
    Prepare the data for being formatted or displayed as a chart.

    Metrics that have more than one value are aggregated using the given aggregation function.
    """
    all_benchmarks = sorted(list(set(baseline.keys()) | set(candidate.keys())))
    baseline_series = []
    candidate_series = []
    for bm in all_benchmarks:
        baseline_series.append(aggregate(baseline[bm][metric]) if bm in baseline and metric in baseline[bm] else None)
        candidate_series.append(aggregate(candidate[bm][metric]) if bm in candidate and metric in candidate[bm] else None)
    return (all_benchmarks, baseline_series, candidate_series)

def main(argv):
    parser = argparse.ArgumentParser(
        prog='compare-benchmarks',
        description='Compare the results of two sets of benchmarks in LNT format.',
        epilog='This script requires the `tabulate` and the `plotly` Python modules.')
    parser.add_argument('baseline', type=argparse.FileType('r'),
        help='Path to a LNT format file containing the benchmark results for the baseline.')
    parser.add_argument('candidate', type=argparse.FileType('r'),
        help='Path to a LNT format file containing the benchmark results for the candidate.')
    parser.add_argument('--output', '-o', type=pathlib.Path, required=False,
        help='Path of a file where to output the resulting comparison. If the output format is `text`, '
             'default to stdout. If the output format is `chart`, default to a temporary file which is '
             'opened automatically once generated, but not removed after creation.')
    parser.add_argument('--metric', type=str, default='execution_time',
        help='The metric to compare. LNT data may contain multiple metrics (e.g. code size, execution time, etc) -- '
             'this option allows selecting which metric is being analyzed. The default is `execution_time`.')
    parser.add_argument('--filter', type=str, required=False,
        help='An optional regular expression used to filter the benchmarks included in the comparison. '
             'Only benchmarks whose names match the regular expression will be included.')
    parser.add_argument('--format', type=str, choices=['text', 'chart'], default='text',
        help='Select the output format. `text` generates a plain-text comparison in tabular form, and `chart` '
             'generates a self-contained HTML graph that can be opened in a browser. The default is `text`.')
    parser.add_argument('--open', action='store_true',
        help='Whether to automatically open the generated HTML file when finished. This option only makes sense '
             'when the output format is `chart`.')
    parser.add_argument('--baseline-name', type=str, default='Baseline',
        help='Optional name to use for the "baseline" label.')
    parser.add_argument('--candidate-name', type=str, default='Candidate',
        help='Optional name to use for the "candidate" label.')
    parser.add_argument('--subtitle', type=str, required=False,
        help='Optional subtitle to use for the chart. This can be used to help identify the contents of the chart. '
             'This option cannot be used with the plain text output.')
    args = parser.parse_args(argv)

    if args.format == 'text' and args.subtitle is not None:
        parser.error('Passing --subtitle makes no sense with --format=text')

    if args.format == 'text' and args.open:
        parser.error('Passing --open makes no sense with --format=text')

    baseline = parse_lnt(args.baseline.readlines())
    candidate = parse_lnt(args.candidate.readlines())

    if args.filter is not None:
        regex = re.compile(args.filter)
        baseline = {k: v for (k, v) in baseline.items() if regex.search(k)}
        candidate = {k: v for (k, v) in candidate.items() if regex.search(k)}

    (benchmarks, baseline_series, candidate_series) = prepare_series(baseline, candidate, args.metric)

    if args.format == 'chart':
        figure = create_chart(benchmarks, baseline_series, candidate_series, subtitle=args.subtitle,
                                                                             baseline_name=args.baseline_name,
                                                                             candidate_name=args.candidate_name)
        do_open = args.output is None or args.open
        output = args.output or tempfile.NamedTemporaryFile(suffix='.html').name
        plotly.io.write_html(figure, file=output, auto_open=do_open)
    else:
        diff = plain_text_comparison(benchmarks, baseline_series, candidate_series, baseline_name=args.baseline_name,
                                                                                    candidate_name=args.candidate_name)
        diff += '\n'
        if args.output is not None:
            with open(args.output, 'w') as out:
                out.write(diff)
        else:
            sys.stdout.write(diff)

if __name__ == '__main__':
    main(sys.argv[1:])