utils/analyzer/SATestBenchmark.py

"""
Static Analyzer qualification infrastructure.

This source file contains all the functionality related to benchmarking
the analyzer on a set projects.  Right now, this includes measuring
execution time and peak memory usage.  Benchmark runs analysis on every
project multiple times to get a better picture about the distribution
of measured values.

Additionally, this file includes a comparison routine for two benchmarking
results that plots the result together on one chart.
"""

import SATestUtils as utils
from SATestBuild import ProjectTester, stdout, TestInfo
from ProjectMap import ProjectInfo

import pandas as pd
from typing import List, Tuple


INDEX_COLUMN = "index"


def _save(data: pd.DataFrame, file_path: str):
    data.to_csv(file_path, index_label=INDEX_COLUMN)


def _load(file_path: str) -> pd.DataFrame:
    return pd.read_csv(file_path, index_col=INDEX_COLUMN)


class Benchmark:
    """
    Becnhmark class encapsulates one functionality: it runs the analysis
    multiple times for the given set of projects and stores results in the
    specified file.
    """
    def __init__(self, projects: List[ProjectInfo], iterations: int,
                 output_path: str):
        self.projects = projects
        self.iterations = iterations
        self.out = output_path

    def run(self):
        results = [self._benchmark_project(project)
                   for project in self.projects]

        data = pd.concat(results, ignore_index=True)
        _save(data, self.out)

    def _benchmark_project(self, project: ProjectInfo) -> pd.DataFrame:
        if not project.enabled:
            stdout(f" \n\n--- Skipping disabled project {project.name}\n")
            return

        stdout(f" \n\n--- Benchmarking project {project.name}\n")

        test_info = TestInfo(project)
        tester = ProjectTester(test_info, silent=True)
        project_dir = tester.get_project_dir()
        output_dir = tester.get_output_dir()

        raw_data = []

        for i in range(self.iterations):
            stdout(f"Iteration #{i + 1}")
            time, mem = tester.build(project_dir, output_dir)
            raw_data.append({"time": time, "memory": mem,
                             "iteration": i, "project": project.name})
            stdout(f"time: {utils.time_to_str(time)}, "
                   f"peak memory: {utils.memory_to_str(mem)}")

        return pd.DataFrame(raw_data)


def compare(old_path: str, new_path: str, plot_file: str):
    """
    Compare two benchmarking results stored as .csv files
    and produce a plot in the specified file.
    """
    old = _load(old_path)
    new = _load(new_path)

    old_projects = set(old["project"])
    new_projects = set(new["project"])
    common_projects = old_projects & new_projects

    # Leave only rows for projects common to both dataframes.
    old = old[old["project"].isin(common_projects)]
    new = new[new["project"].isin(common_projects)]

    old, new = _normalize(old, new)

    # Seaborn prefers all the data to be in one dataframe.
    old["kind"] = "old"
    new["kind"] = "new"
    data = pd.concat([old, new], ignore_index=True)

    # TODO: compare data in old and new dataframes using statistical tests
    #       to check if they belong to the same distribution
    _plot(data, plot_file)


def _normalize(old: pd.DataFrame,
               new: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
    # This creates a dataframe with all numerical data averaged.
    means = old.groupby("project").mean()
    return _normalize_impl(old, means), _normalize_impl(new, means)


def _normalize_impl(data: pd.DataFrame, means: pd.DataFrame):
    # Right now 'means' has one row corresponding to one project,
    # while 'data' has N rows for each project (one for each iteration).
    #
    # In order for us to work easier with this data, we duplicate
    # 'means' data to match the size of the 'data' dataframe.
    #
    # All the columns from 'data' will maintain their names, while
    # new columns coming from 'means' will have "_mean" suffix.
    joined_data = data.merge(means, on="project", suffixes=("", "_mean"))
    _normalize_key(joined_data, "time")
    _normalize_key(joined_data, "memory")
    return joined_data


def _normalize_key(data: pd.DataFrame, key: str):
    norm_key = _normalized_name(key)
    mean_key = f"{key}_mean"
    data[norm_key] = data[key] / data[mean_key]


def _normalized_name(name: str) -> str:
    return f"normalized {name}"


def _plot(data: pd.DataFrame, plot_file: str):
    import matplotlib
    import seaborn as sns
    from matplotlib import pyplot as plt

    sns.set_style("whitegrid")
    # We want to have time and memory charts one above the other.
    figure, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6))

    def _subplot(key: str, ax: matplotlib.axes.Axes):
        sns.boxplot(x="project", y=_normalized_name(key), hue="kind",
                    data=data, palette=sns.color_palette("BrBG", 2), ax=ax)

    _subplot("time", ax1)
    # No need to have xlabels on both top and bottom charts.
    ax1.set_xlabel("")

    _subplot("memory", ax2)
    # The legend on the top chart is enough.
    ax2.get_legend().remove()

    figure.savefig(plot_file)