| """ | |
| Static Analyzer qualification infrastructure. | |
| This source file contains all the functionality related to benchmarking | |
| the analyzer on a set projects. Right now, this includes measuring | |
| execution time and peak memory usage. Benchmark runs analysis on every | |
| project multiple times to get a better picture about the distribution | |
| of measured values. | |
| Additionally, this file includes a comparison routine for two benchmarking | |
| results that plots the result together on one chart. | |
| """ | |
| import SATestUtils as utils | |
| from SATestBuild import ProjectTester, stdout, TestInfo | |
| from ProjectMap import ProjectInfo | |
| import pandas as pd | |
| from typing import List, Tuple | |
| INDEX_COLUMN = "index" | |
| def _save(data: pd.DataFrame, file_path: str): | |
| data.to_csv(file_path, index_label=INDEX_COLUMN) | |
| def _load(file_path: str) -> pd.DataFrame: | |
| return pd.read_csv(file_path, index_col=INDEX_COLUMN) | |
| class Benchmark: | |
| """ | |
| Becnhmark class encapsulates one functionality: it runs the analysis | |
| multiple times for the given set of projects and stores results in the | |
| specified file. | |
| """ | |
| def __init__(self, projects: List[ProjectInfo], iterations: int, output_path: str): | |
| self.projects = projects | |
| self.iterations = iterations | |
| self.out = output_path | |
| def run(self): | |
| results = [self._benchmark_project(project) for project in self.projects] | |
| data = pd.concat(results, ignore_index=True) | |
| _save(data, self.out) | |
| def _benchmark_project(self, project: ProjectInfo) -> pd.DataFrame: | |
| if not project.enabled: | |
| stdout(f" \n\n--- Skipping disabled project {project.name}\n") | |
| return | |
| stdout(f" \n\n--- Benchmarking project {project.name}\n") | |
| test_info = TestInfo(project) | |
| tester = ProjectTester(test_info, silent=True) | |
| project_dir = tester.get_project_dir() | |
| output_dir = tester.get_output_dir() | |
| raw_data = [] | |
| for i in range(self.iterations): | |
| stdout(f"Iteration #{i + 1}") | |
| time, mem = tester.build(project_dir, output_dir) | |
| raw_data.append( | |
| {"time": time, "memory": mem, "iteration": i, "project": project.name} | |
| ) | |
| stdout( | |
| f"time: {utils.time_to_str(time)}, " | |
| f"peak memory: {utils.memory_to_str(mem)}" | |
| ) | |
| return pd.DataFrame(raw_data) | |
| def compare(old_path: str, new_path: str, plot_file: str): | |
| """ | |
| Compare two benchmarking results stored as .csv files | |
| and produce a plot in the specified file. | |
| """ | |
| old = _load(old_path) | |
| new = _load(new_path) | |
| old_projects = set(old["project"]) | |
| new_projects = set(new["project"]) | |
| common_projects = old_projects & new_projects | |
| # Leave only rows for projects common to both dataframes. | |
| old = old[old["project"].isin(common_projects)] | |
| new = new[new["project"].isin(common_projects)] | |
| old, new = _normalize(old, new) | |
| # Seaborn prefers all the data to be in one dataframe. | |
| old["kind"] = "old" | |
| new["kind"] = "new" | |
| data = pd.concat([old, new], ignore_index=True) | |
| # TODO: compare data in old and new dataframes using statistical tests | |
| # to check if they belong to the same distribution | |
| _plot(data, plot_file) | |
| def _normalize( | |
| old: pd.DataFrame, new: pd.DataFrame | |
| ) -> Tuple[pd.DataFrame, pd.DataFrame]: | |
| # This creates a dataframe with all numerical data averaged. | |
| means = old.groupby("project").mean() | |
| return _normalize_impl(old, means), _normalize_impl(new, means) | |
| def _normalize_impl(data: pd.DataFrame, means: pd.DataFrame): | |
| # Right now 'means' has one row corresponding to one project, | |
| # while 'data' has N rows for each project (one for each iteration). | |
| # | |
| # In order for us to work easier with this data, we duplicate | |
| # 'means' data to match the size of the 'data' dataframe. | |
| # | |
| # All the columns from 'data' will maintain their names, while | |
| # new columns coming from 'means' will have "_mean" suffix. | |
| joined_data = data.merge(means, on="project", suffixes=("", "_mean")) | |
| _normalize_key(joined_data, "time") | |
| _normalize_key(joined_data, "memory") | |
| return joined_data | |
| def _normalize_key(data: pd.DataFrame, key: str): | |
| norm_key = _normalized_name(key) | |
| mean_key = f"{key}_mean" | |
| data[norm_key] = data[key] / data[mean_key] | |
| def _normalized_name(name: str) -> str: | |
| return f"normalized {name}" | |
| def _plot(data: pd.DataFrame, plot_file: str): | |
| import matplotlib | |
| import seaborn as sns | |
| from matplotlib import pyplot as plt | |
| sns.set_style("whitegrid") | |
| # We want to have time and memory charts one above the other. | |
| figure, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6)) | |
| def _subplot(key: str, ax: matplotlib.axes.Axes): | |
| sns.boxplot( | |
| x="project", | |
| y=_normalized_name(key), | |
| hue="kind", | |
| data=data, | |
| palette=sns.color_palette("BrBG", 2), | |
| ax=ax, | |
| ) | |
| _subplot("time", ax1) | |
| # No need to have xlabels on both top and bottom charts. | |
| ax1.set_xlabel("") | |
| _subplot("memory", ax2) | |
| # The legend on the top chart is enough. | |
| ax2.get_legend().remove() | |
| figure.savefig(plot_file) |