[𝘀𝗽𝗿] initial version Created using spr 1.3.7
diff --git a/lnt/lnttool/__init__.py b/lnt/lnttool/__init__.py index f93a9c1..a949161 100644 --- a/lnt/lnttool/__init__.py +++ b/lnt/lnttool/__init__.py
@@ -13,6 +13,7 @@ from .runtest import group_runtest from .send_daily_report import action_send_daily_report from .send_run_comparison import action_send_run_comparison +from .expire_abtests import action_expire_abtests from .showtests import action_showtests from .submit import action_submit from .updatedb import action_updatedb @@ -45,6 +46,7 @@ main.add_command(action_checkformat) main.add_command(action_convert) main.add_command(action_create) +main.add_command(action_expire_abtests) main.add_command(action_import) main.add_command(action_importreport) main.add_command(action_profile)
diff --git a/lnt/lnttool/expire_abtests.py b/lnt/lnttool/expire_abtests.py new file mode 100644 index 0000000..33c369d --- /dev/null +++ b/lnt/lnttool/expire_abtests.py
@@ -0,0 +1,93 @@ +import contextlib +import datetime +import re + +import click + + +def _parse_age(value): + """Parse an age string like '90d', '4w', '6m', '1y' into a cutoff datetime.""" + m = re.fullmatch(r'(\d+)([dwmy])', value) + if not m: + raise click.BadParameter( + "expected a positive integer followed by d/w/m/y " + "(e.g. 90d, 4w, 6m, 1y)", + param_hint="'--older-than'") + n, unit = int(m.group(1)), m.group(2) + days = {'d': n, 'w': n * 7, 'm': n * 30, 'y': n * 365}[unit] + return datetime.datetime.utcnow() - datetime.timedelta(days=days) + + +@click.command("expire-abtests") +@click.argument("instance_path", type=click.UNPROCESSED) +@click.option("--database", default="default", show_default=True, + help="database to expire experiments from") +@click.option("--testsuite", "-s", default="nts", show_default=True, + help="testsuite to expire experiments from") +@click.option("--older-than", "older_than", required=True, + help="delete experiments older than this age (e.g. 90d, 4w, 6m, 1y)") +@click.option("--dry-run", is_flag=True, + help="print what would be deleted without making any changes") +def action_expire_abtests(instance_path, database, testsuite, + older_than, dry_run): + """Delete unpinned A/B experiments older than a given age. + +\b +Removes unpinned ABExperiment records (and their associated ABRun and +ABSample rows) whose creation time predates the specified age threshold. +Pinned ('Keep Forever') experiments are never deleted. + +\b +Age format: a positive integer followed by a unit: + d days (e.g. 90d) + w weeks (e.g. 4w) + m months (approx 30 days each, e.g. 6m) + y years (approx 365 days each, e.g. 1y) + """ + import lnt.server.instance + + cutoff = _parse_age(older_than) + + instance = lnt.server.instance.Instance.frompath(instance_path) + with contextlib.closing(instance.get_database(database)) as db: + session = db.make_session() + ts = db.testsuite[testsuite] + + to_delete = ( + session.query(ts.ABExperiment) + .filter(ts.ABExperiment.created_time < cutoff, + ts.ABExperiment.pinned == False) # noqa: E712 + .all()) + + if not to_delete: + click.echo("No experiments to delete.") + return + + for exp in to_delete: + click.echo("%s experiment #%d: %s" % ( + "Would delete" if dry_run else "Deleting", + exp.id, exp.name or "(unnamed)")) + + if dry_run: + return + + # Delete ABSample and ABRun children before the ABExperiment rows to + # respect foreign-key constraints. + run_ids = [rid for exp in to_delete + for rid in (exp.control_run_id, exp.variant_run_id) + if rid is not None] + + session.query(ts.ABSample) \ + .filter(ts.ABSample.run_id.in_(run_ids)) \ + .delete(synchronize_session=False) + + session.query(ts.ABRun) \ + .filter(ts.ABRun.id.in_(run_ids)) \ + .delete(synchronize_session=False) + + for exp in to_delete: + session.delete(exp) + + session.commit() + click.echo("Deleted %d experiment%s." % + (len(to_delete), "s" if len(to_delete) != 1 else ""))
diff --git a/lnt/server/db/migrations/new_suite.py b/lnt/server/db/migrations/new_suite.py index e627e06..23c80fd 100644 --- a/lnt/server/db/migrations/new_suite.py +++ b/lnt/server/db/migrations/new_suite.py
@@ -2,6 +2,7 @@ from . import upgrade_2_to_3 from . import upgrade_7_to_8 from . import upgrade_8_to_9 +from . import upgrade_18_to_19 def init_new_testsuite(engine, session, name): @@ -17,3 +18,6 @@ session.commit() upgrade_8_to_9.upgrade_testsuite(engine, session, name) session.commit() + ts_defn = session.query(upgrade_0_to_1.TestSuite).filter_by(name=name).first() + upgrade_18_to_19.upgrade_testsuite(engine, ts_defn.db_key_name) + session.commit()
diff --git a/lnt/server/db/migrations/upgrade_18_to_19.py b/lnt/server/db/migrations/upgrade_18_to_19.py new file mode 100644 index 0000000..08aa7d9 --- /dev/null +++ b/lnt/server/db/migrations/upgrade_18_to_19.py
@@ -0,0 +1,107 @@ +"""Add ABRun, ABSample, and ABExperiment tables for A/B performance testing. + +ABRun intentionally omits order_id so that A/B test runs never participate +in trend analysis or FieldChange/Regression detection. + +This migration creates the AB tables with their dynamic run/sample field +columns following the same column-type rules as upgrade_0_to_1. +""" + +import sqlalchemy +from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, + LargeBinary, String, select) +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base + +import lnt.server.db.migrations.upgrade_0_to_1 as upgrade_0_to_1 +from lnt.server.db.migrations.util import introspect_table + + +def _add_ab_tables(test_suite): + """Return a Base with ABRun, ABSample, and ABExperiment for test_suite. + + Machine and Test stubs are included in the same Base so that FK + references resolve during create_all.""" + db_key_name = test_suite.db_key_name + Base = declarative_base() + + class Machine(Base): + __tablename__ = db_key_name + '_Machine' + __table_args__ = {'extend_existing': True} + id = Column("ID", Integer, primary_key=True) + + class Test(Base): + __tablename__ = db_key_name + '_Test' + __table_args__ = {'extend_existing': True} + id = Column("ID", Integer, primary_key=True) + + class ABRun(Base): + __tablename__ = db_key_name + '_ABRun' + id = Column("ID", Integer, primary_key=True) + machine_id = Column("MachineID", Integer, ForeignKey(Machine.id), + index=True) + start_time = Column("StartTime", DateTime) + end_time = Column("EndTime", DateTime) + parameters_data = Column("Parameters", LargeBinary) + + class_dict = locals() + for item in test_suite.run_fields: + class_dict[item.name] = Column(item.name, String(256)) + + class ABSample(Base): + __tablename__ = db_key_name + '_ABSample' + id = Column("ID", Integer, primary_key=True) + run_id = Column("RunID", Integer, ForeignKey(ABRun.id), index=True) + test_id = Column("TestID", Integer, ForeignKey(Test.id), index=True) + + class_dict = locals() + for item in test_suite.sample_fields: + if item.type.name == 'Real': + class_dict[item.name] = Column(item.name, Float) + elif item.type.name == 'Status': + class_dict[item.name] = Column( + item.name, Integer, + ForeignKey(upgrade_0_to_1.StatusKind.id)) + elif item.type.name == 'Hash': + class_dict[item.name] = Column(item.name, String) + + class ABExperiment(Base): + __tablename__ = db_key_name + '_ABExperiment' + id = Column("ID", Integer, primary_key=True) + name = Column("Name", String(256)) + created_time = Column("CreatedTime", DateTime) + extra = Column("Extra", String) + pinned = Column("Pinned", Boolean, default=False) + control_run_id = Column("ControlRunID", Integer, + ForeignKey(ABRun.id)) + variant_run_id = Column("VariantRunID", Integer, + ForeignKey(ABRun.id)) + + return Base + + +def upgrade_testsuite(engine, db_key_name): + """Create the AB tables for a single test suite identified by db_key_name.""" + session = sessionmaker(engine)() + try: + test_suite = session.query(upgrade_0_to_1.TestSuite).filter_by( + db_key_name=db_key_name).first() + if test_suite is None: + return + Base = _add_ab_tables(test_suite) + # Only create new tables; use checkfirst=True (the default) so + # existing tables are left untouched. + Base.metadata.create_all(engine, checkfirst=True) + finally: + session.close() + + +def upgrade(engine): + """Create AB tables for every existing test suite.""" + test_suite_table = introspect_table(engine, 'TestSuite') + + with engine.begin() as trans: + suites = list(trans.execute(select([test_suite_table.c.DBKeyName]))) + + for (db_key_name,) in suites: + upgrade_testsuite(engine, db_key_name)
diff --git a/lnt/server/db/testsuitedb.py b/lnt/server/db/testsuitedb.py index 3dc6419..a088c60 100644 --- a/lnt/server/db/testsuitedb.py +++ b/lnt/server/db/testsuitedb.py
@@ -13,7 +13,7 @@ import aniso8601 import sqlalchemy import flask -from sqlalchemy import Float, String, Integer, Column, ForeignKey, Binary, DateTime +from sqlalchemy import Float, String, Integer, Column, ForeignKey, Binary, Boolean, DateTime from sqlalchemy.orm import relation from sqlalchemy.orm.exc import ObjectDeletedError from lnt.util import logger @@ -756,6 +756,81 @@ def __str__(self): return "Baseline({})".format(self.name) + # A/B testing tables. ABRun intentionally omits order_id so that A/B + # runs never participate in trend analysis or FieldChange/Regression + # detection. + + class ABRun(self.base, ParameterizedMixin): + __tablename__ = db_key_name + '_ABRun' + + fields = self.run_fields + id = Column("ID", Integer, primary_key=True) + machine_id = Column("MachineID", Integer, ForeignKey(Machine.id), + index=True) + start_time = Column("StartTime", DateTime) + end_time = Column("EndTime", DateTime) + parameters_data = Column("Parameters", Binary, index=False, + unique=False) + + machine = relation(Machine) + + # Dynamic run-field columns. Create fresh Column objects rather + # than reusing item.column, which points to the Run table. + class_dict = locals() + for item in fields: + class_dict[item.name] = testsuite.make_run_column(item.name) + + @property + def parameters(self): + return dict(json.loads(self.parameters_data)) + + @parameters.setter + def parameters(self, data): + self.parameters_data = json.dumps( + sorted(data.items())).encode("utf-8") + + class ABSample(self.base, ParameterizedMixin): + __tablename__ = db_key_name + '_ABSample' + + fields = self.sample_fields + id = Column("ID", Integer, primary_key=True) + run_id = Column("RunID", Integer, ForeignKey(ABRun.id), index=True) + test_id = Column("TestID", Integer, ForeignKey(Test.id), + index=True) + + run = relation(ABRun) + test = relation(Test) + + # Dynamic sample-field columns. Create fresh Column objects rather + # than reusing item.column, which points to the Sample table. + class_dict = locals() + for item in fields: + class_dict[item.name] = testsuite.make_sample_column( + item.name, item.type.name) + + ABRun.ab_samples = relation(ABSample, back_populates='run', + cascade="all, delete-orphan") + + class ABExperiment(self.base, ParameterizedMixin): + __tablename__ = db_key_name + '_ABExperiment' + + fields = [] + id = Column("ID", Integer, primary_key=True) + name = Column("Name", String(256)) + created_time = Column("CreatedTime", DateTime) + extra = Column("Extra", String) + pinned = Column("Pinned", Boolean, default=False) + + # Two FK references to ABRun; foreign_keys disambiguates them. + control_run_id = Column("ControlRunID", Integer, + ForeignKey(ABRun.id)) + variant_run_id = Column("VariantRunID", Integer, + ForeignKey(ABRun.id)) + control_run = relation(ABRun, + foreign_keys=[control_run_id]) + variant_run = relation(ABRun, + foreign_keys=[variant_run_id]) + self.Machine = Machine self.Run = Run self.Test = Test @@ -767,10 +842,15 @@ self.RegressionIndicator = RegressionIndicator self.ChangeIgnore = ChangeIgnore self.Baseline = Baseline + self.ABRun = ABRun + self.ABSample = ABSample + self.ABExperiment = ABExperiment # Create the compound index we cannot declare inline. sqlalchemy.schema.Index("ix_%s_Sample_RunID_TestID" % db_key_name, Sample.run_id, Sample.test_id) + sqlalchemy.schema.Index("ix_%s_ABSample_RunID_TestID" % db_key_name, + ABSample.run_id, ABSample.test_id) def create_tables(self, engine): self.base.metadata.create_all(engine) @@ -1125,6 +1205,99 @@ self._importSampleValues(session, data['tests'], run, config) return run + def _importABRun(self, session, run_data, machine): + """Create and insert an ABRun for the given machine. + + No order tracking is performed; A/B runs are isolated from trend + analysis and regression detection. + """ + run_parameters = run_data.copy() + run_parameters.pop('id', None) + run_parameters.pop('order_by', None) + run_parameters.pop('order_id', None) + run_parameters.pop('machine_id', None) + + start_time_str = run_parameters.pop('start_time', None) + if start_time_str: + try: + start_time = aniso8601.parse_datetime(start_time_str) + except ValueError: + start_time = datetime.datetime.strptime(start_time_str, + "%Y-%m-%d %H:%M:%S") + else: + start_time = None + + end_time_str = run_parameters.pop('end_time', None) + if end_time_str: + try: + end_time = aniso8601.parse_datetime(end_time_str) + except ValueError: + end_time = datetime.datetime.strptime(end_time_str, + "%Y-%m-%d %H:%M:%S") + else: + end_time = None + + run = self.ABRun() + run.machine = machine + run.start_time = start_time + run.end_time = end_time + for item in self.run_fields: + value = run_parameters.pop(item.name, None) + run.set_field(item, value) + run.parameters = run_parameters + session.add(run) + return run + + def _importABSampleValues(self, session, tests_data, run): + """Create ABSample rows for the given tests data and ABRun. + + Mirrors _importSampleValues but creates ABSample objects and skips + profile handling. + """ + test_cache = dict((test.name, test) + for test in session.query(self.Test)) + field_dict = dict([(f.name, f) for f in self.sample_fields]) + all_samples = [] + + for test_data in tests_data: + name = test_data['name'] + test = test_cache.get(name) + if test is None: + test = self.Test(test_data['name']) + test_cache[name] = test + session.add(test) + + samples = [] + for key, values in test_data.items(): + if key == 'name' or key == 'id' or key.endswith('_id'): + continue + field = field_dict.get(key) + if field is None: + raise ValueError("test %s: Metric %r unknown in suite" % + (name, key)) + if not isinstance(values, list): + values = [values] + while len(samples) < len(values): + sample = self.ABSample() + sample.run = run + sample.test = test + samples.append(sample) + all_samples.append(sample) + for sample, value in zip(samples, values): + sample.set_field(field, value) + + session.add_all(all_samples) + + def importABDataFromDict(self, session, data): + """Import a report into ABRun/ABSample tables. + + No order or regression tracking is performed. + """ + machine = self._getOrCreateMachine(session, data['machine'], 'match') + run = self._importABRun(session, data['run'], machine) + self._importABSampleValues(session, data['tests'], run) + return run + # Simple query support (mostly used by templates) def machines(self, session, name=None):
diff --git a/lnt/server/reporting/analysis.py b/lnt/server/reporting/analysis.py index 8d18ec0..565d1ae 100644 --- a/lnt/server/reporting/analysis.py +++ b/lnt/server/reporting/analysis.py
@@ -419,3 +419,29 @@ self.profile_map[(run_id, test_id)] = profile_id self.loaded_run_ids |= to_load + + +class ABRunInfo(RunInfo): + """Like RunInfo but queries ABSample instead of Sample. + + Uses getattr(ts.ABSample, f.name) to reference columns rather than + f.column, which points to the Sample table. + """ + + def _load_samples_for_runs(self, session, run_ids, only_tests): + to_load = set(run_ids) - self.loaded_run_ids + if not to_load: + return + + ABSample = self.testsuite.ABSample + columns = [ABSample.run_id, ABSample.test_id] + columns.extend( + getattr(ABSample, f.name) for f in self.testsuite.sample_fields) + q = session.query(*columns) + if only_tests: + q = q.filter(ABSample.test_id.in_(only_tests)) + q = q.filter(ABSample.run_id.in_(to_load)) + for (run_id, test_id, *sample_values) in q: + self.sample_map[(run_id, test_id)] = sample_values + + self.loaded_run_ids |= to_load
diff --git a/lnt/server/ui/api.py b/lnt/server/ui/api.py index 00119c9..b1998ec 100644 --- a/lnt/server/ui/api.py +++ b/lnt/server/ui/api.py
@@ -1,3 +1,4 @@ +import datetime import lnt.util.ImportData import re import yaml @@ -9,6 +10,7 @@ from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound +from lnt.server.reporting.analysis import ABRunInfo from lnt.server.ui.util import convert_revision from lnt.server.ui.decorators import in_db from lnt.testing import PASS @@ -589,6 +591,143 @@ return results +def _ab_exp_to_dict(exp): + return { + 'id': exp.id, + 'name': exp.name, + 'created_time': str(exp.created_time), + 'pinned': exp.pinned, + 'control_run_id': exp.control_run_id, + 'variant_run_id': exp.variant_run_id, + } + + +class ABTests(Resource): + """List or create A/B experiments for a test suite.""" + method_decorators = [in_db] + + @staticmethod + @requires_auth_token + def post(): + """Submit control and variant reports and create an ABExperiment.""" + session = request.session + ts = request.get_testsuite() + body = request.get_json(force=True) + if body is None: + abort(400, msg="Request body must be JSON.") + for key in ('control', 'variant'): + if key not in body: + abort(400, msg="Missing required field: %r" % key) + control_run = ts.importABDataFromDict(session, body['control']) + session.flush() + variant_run = ts.importABDataFromDict(session, body['variant']) + session.flush() + exp = ts.ABExperiment() + exp.name = body.get('name', '') + exp.created_time = datetime.datetime.utcnow() + exp.control_run_id = control_run.id + exp.variant_run_id = variant_run.id + exp.pinned = bool(body.get('pinned', False)) + session.add(exp) + session.commit() + new_url = ('%sapi/db_%s/v4/%s/abtest/%s' % + (request.url_root, g.db_name, g.testsuite_name, exp.id)) + result = _ab_exp_to_dict(exp) + result['url'] = new_url + response = jsonify(result) + response.status = '201' + response.headers.add('Location', new_url) + return response + + @staticmethod + def get(): + """List A/B experiments, optionally filtered by pinned status.""" + session = request.session + ts = request.get_testsuite() + q = session.query(ts.ABExperiment).order_by( + ts.ABExperiment.created_time.desc()) + pinned = request.args.get('pinned') + if pinned is not None: + q = q.filter(ts.ABExperiment.pinned == (pinned == 'true')) + limit = int(request.args.get('limit', 50)) + offset = int(request.args.get('offset', 0)) + exps = q.limit(limit).offset(offset).all() + return {'experiments': [_ab_exp_to_dict(e) for e in exps]} + + +class ABTestDetail(Resource): + """Retrieve or update a single A/B experiment.""" + method_decorators = [in_db] + + @staticmethod + def get(abtest_id): + """Return the experiment metadata and per-test comparison results.""" + session = request.session + ts = request.get_testsuite() + exp = session.query(ts.ABExperiment).filter_by(id=abtest_id).first() + if exp is None: + abort(404, msg="No such A/B experiment.") + + control_run = session.query(ts.ABRun).filter_by( + id=exp.control_run_id).first() + variant_run = session.query(ts.ABRun).filter_by( + id=exp.variant_run_id).first() + if control_run is None or variant_run is None: + abort(404, msg="Experiment references a missing ABRun.") + + sri = ABRunInfo(session, ts, + [exp.control_run_id, exp.variant_run_id]) + test_ids = sri.test_ids + test_rows = session.query(ts.Test.id, ts.Test.name).filter( + ts.Test.id.in_(test_ids)).all() + test_name_map = {tid: name for tid, name in test_rows} + + metric_fields = [f for f in ts.sample_fields + if f.type.name in ('Real', 'Integer')] + + comparisons = [] + for test_id in sorted(test_ids): + test_name = test_name_map.get(test_id, str(test_id)) + for field in metric_fields: + cr = sri.get_run_comparison_result( + variant_run, control_run, test_id, field, None) + if cr.current is None and cr.previous is None: + continue + comparisons.append({ + 'test_name': test_name, + 'field': field.name, + 'control': cr.previous, + 'variant': cr.current, + 'pct_delta': round(cr.pct_delta * 100, 4) + if cr.pct_delta is not None else None, + 'status': cr.get_value_status(), + }) + + return { + 'experiment': _ab_exp_to_dict(exp), + 'comparisons': comparisons, + } + + @staticmethod + @requires_auth_token + def patch(abtest_id): + """Update mutable fields (currently: pinned) of an ABExperiment.""" + session = request.session + ts = request.get_testsuite() + exp = session.query(ts.ABExperiment).filter_by(id=abtest_id).first() + if exp is None: + abort(404, msg="No such A/B experiment.") + body = request.get_json(force=True) + if body is None: + abort(400, msg="Request body must be JSON.") + if 'pinned' in body: + exp.pinned = bool(body['pinned']) + if 'name' in body: + exp.name = body['name'] + session.commit() + return _ab_exp_to_dict(exp) + + def ts_path(path): """Make a URL path with a database and test suite embedded in them.""" return "/api/db_<string:db>/v4/<string:ts>/" + path @@ -619,3 +758,5 @@ regression_url = \ "regression/<int:machine_id>/<int:test_id>/<int:field_index>" api.add_resource(Regression, ts_path(regression_url)) + api.add_resource(ABTests, ts_path("abtest"), ts_path("abtest/")) + api.add_resource(ABTestDetail, ts_path("abtest/<int:abtest_id>"))
diff --git a/lnt/server/ui/templates/v4_abtest.html b/lnt/server/ui/templates/v4_abtest.html new file mode 100644 index 0000000..eec08ea --- /dev/null +++ b/lnt/server/ui/templates/v4_abtest.html
@@ -0,0 +1,241 @@ +{% import "utils.html" as utils %} + +{% extends "layout.html" %} +{% set components = [(ts.name, v4_url_for(".v4_recent_activity")), + ("A/B Experiments", v4_url_for(".v4_abtests")), + (exp.name or ('Experiment #' ~ exp.id), + v4_url_for(".v4_abtest", id=exp.id))] %} +{% block title %}A/B: {{ exp.name or exp.id }}{% endblock %} + +{% block head %} + <script src="{{ url_for('.static', filename='sorttable.js') }}"></script> + <style> + /* Toggle switch for pin/unpin */ + .pin-switch { position:relative; display:inline-block; width:38px; height:20px; vertical-align:middle; } + .pin-switch input { opacity:0; width:0; height:0; } + .pin-slider { position:absolute; cursor:pointer; top:0; left:0; right:0; bottom:0; + background-color:#ccc; border-radius:20px; transition:.25s; } + .pin-slider:before { position:absolute; content:""; height:14px; width:14px; + left:3px; bottom:3px; background-color:white; border-radius:50%; transition:.25s; } + .pin-switch input:checked + .pin-slider { background-color:#337ab7; } + .pin-switch input:checked + .pin-slider:before { transform:translateX(18px); } + </style> + <script> + function selectAll(source) { + $(source).closest("table").find("input:checkbox").prop("checked", source.checked); + } + </script> +{% endblock %} + +{% block sidebar %} + <ul class="nav nav-list bs-docs-sidenav"> + <li><a href="#run_info"><i class="icon-chevron-right"></i>Run Info</a></li> + <li><a href="#view_options"><i class="icon-chevron-right"></i>View Options</a></li> + {% for field in metric_fields %} + <li><a href="#{{ field.name }}"><i class="icon-chevron-right"></i>{{ field.display_name }}</a></li> + {% endfor %} + </ul> +{% endblock %} + +{% block body %} + +{# ------------------------------------------------------------------ #} +{# Experiment header #} +{# ------------------------------------------------------------------ #} +<h3> + {{ exp.name or '(unnamed experiment)' }} + <small>#{{ exp.id }}</small> +</h3> + +<form method="POST" action="{{ v4_url_for('.v4_abtest_pin', id=exp.id) }}" style="display:inline" id="pin-form"> + <label class="pin-switch" title="Keep Forever"> + <input type="checkbox" onchange="document.getElementById('pin-form').submit()" + {{ 'checked' if exp.pinned else '' }}> + <span class="pin-slider"></span> + </label> +</form> +<small class="text-muted" style="margin-left:8px">Keep Forever — created {{ exp.created_time }}</small> + +<div style="margin-top:1em"></div> +{# ------------------------------------------------------------------ #} +{# Run info: single table, control and variant side by side #} +{# ------------------------------------------------------------------ #} +<section id="run_info"> +{{ utils.render_popup_begin('run_info', 'Run Info', true) }} + +<table class="table table-striped table-condensed table-hover"> + <thead> + <tr> + <th>Field</th> + <th>Control</th> + <th>Variant</th> + </tr> + </thead> + <tbody> + <tr> + <td><b>run</b></td> + <td>{{ control_run.id }}</td> + <td>{{ variant_run.id }}</td> + </tr> + <tr> + <td><b>Machine</b></td> + <td>{{ control_run.machine.name }}</td> + <td>{{ variant_run.machine.name }}</td> + </tr> + {% for item in control_run.machine.fields %} + <tr> + <td><b>{{ item.name }}</b></td> + <td>{{ utils.newline_to_br(control_run.machine.get_field(item)) }}</td> + <td>{{ utils.newline_to_br(variant_run.machine.get_field(item)) }}</td> + </tr> + {% endfor %} + {% for key, value in control_run.machine.parameters|dictsort %} + <tr> + <td><b>{{ key }}</b></td> + <td>{{ utils.newline_to_br(value) }}</td> + <td>{{ utils.newline_to_br(variant_run.machine.parameters.get(key, '')) }}</td> + </tr> + {% endfor %} + {% for key, value in variant_run.machine.parameters|dictsort %} + {% if key not in control_run.machine.parameters %} + <tr> + <td><b>{{ key }}</b></td> + <td></td> + <td>{{ utils.newline_to_br(value) }}</td> + </tr> + {% endif %} + {% endfor %} + <tr> + <td><b>start</b></td> + <td>{{ control_run.start_time }}</td> + <td>{{ variant_run.start_time }}</td> + </tr> + <tr> + <td><b>end</b></td> + <td>{{ control_run.end_time }}</td> + <td>{{ variant_run.end_time }}</td> + </tr> + {% for item in control_run.fields %} + <tr> + <td><b>{{ item.name }}</b></td> + <td>{{ utils.newline_to_br(control_run.get_field(item)) }}</td> + <td>{{ utils.newline_to_br(variant_run.get_field(item)) }}</td> + </tr> + {% endfor %} + </tbody> +</table> + +{{ utils.render_popup_end() }} +</section> + +{# ------------------------------------------------------------------ #} +{# View options #} +{# ------------------------------------------------------------------ #} +<section id="view_options"> +{{ utils.render_popup_begin('view_options', 'View Options', true) }} +<form method="GET"> + <table class="table table-striped table-condensed table-hover"> + <tr> + <td><label for="show_all">Show All Tests</label></td> + <td><input id="show_all" type="checkbox" name="show_all" value="yes" + {{ "checked" if options.show_all else "" }}/></td> + </tr> + <tr> + <td><label for="show_delta">Show Delta</label></td> + <td><input id="show_delta" type="checkbox" name="show_delta" value="yes" + {{ "checked" if options.show_delta else "" }}/></td> + </tr> + <tr> + <td><label for="show_small_diff">Show Small Differences</label></td> + <td><input id="show_small_diff" type="checkbox" name="show_small_diff" value="yes" + {{ "checked" if options.show_small_diff else "" }}/></td> + </tr> + <tr> + <td><label for="test_filter">Test Filter (regexp)</label></td> + <td><input id="test_filter" type="text" name="test_filter" + value="{{ options.test_filter }}"/></td> + </tr> + <tr> + <td colspan="2"><input type="submit" name="submit" value="Update"/></td> + </tr> + </table> +</form> +{{ utils.render_popup_end() }} +</section> + +{# ------------------------------------------------------------------ #} +{# Per-metric result tables #} +{# ------------------------------------------------------------------ #} +{{ utils.regex_filter_box(input_id='filter', + selector='.searchable tr', + placeholder="Benchmark name regex...", + selector_part_to_search=".benchmark-name") }} + +<form id="ab_scurve_form" method="GET" + action="{{ v4_url_for('.v4_abtest_scurve', id=exp.id) }}"> + +{% for field in metric_fields %} + {% set field_rows = [] %} + {% for row in comparisons %} + {% if row.field.name == field.name %} + {% set cr = row.cr %} + {% set value_status = cr.get_value_status(ignore_small=not options.show_small_diff) %} + {% if options.show_all or value_status in (analysis.REGRESSED, analysis.IMPROVED) %} + {% if field_rows.append(row) %}{% endif %} + {% endif %} + {% endif %} + {% endfor %} + + {% if field_rows %} + {% set unit_suffix = ' (%s)' % field.unit if field.unit else '' %} + <section id="{{ field.name }}"> + {{ utils.render_popup_begin('ab_data-' ~ field.name, + field.display_name ~ unit_suffix, false) }} + <table class="table table-striped table-hover table-condensed sortable searchable"> + <thead> + <tr> + <th class="sorttable_nosort"> + <input type="checkbox" id="select-all-{{ field.name }}" + onClick="selectAll(this)"/> + </th> + <th>Benchmark</th> + <th>Control</th> + <th>Variant</th> + <th>Δ</th> + {% if options.show_delta %}<th>Δ abs</th>{% endif %} + <th>σ</th> + <th>N</th> + </tr> + </thead> + <tbody> + {% for row in field_rows %} + {% set cr = row.cr %} + <tr> + <td><input type="checkbox" name="plot" + value="{{ row.test_id }}.{{ row.field.name }}"/></td> + <td class="benchmark-name">{{ row.test_name }}</td> + <td class="right">{{ '%.4f' % cr.previous if cr.previous is not none else '-' }}</td> + <td class="right">{{ '%.4f' % cr.current if cr.current is not none else '-' }}</td> + {{ cr.pct_delta|aspctcell(reverse=cr.bigger_is_better)|safe }} + {% if options.show_delta %} + <td class="right">{{ '%.4f' % cr.delta if cr.delta else '-' }}</td> + {% endif %} + <td class="right">{{ '%.4f' % cr.stddev if cr.stddev is not none else '-' }}</td> + <td class="right">{{ cr.samples|length }}</td> + </tr> + {% endfor %} + </tbody> + </table> + {{ utils.render_popup_end() }} + </section> + {% endif %} +{% endfor %} + +{% if not comparisons %} +<p>No comparison data available.</p> +{% endif %} + +<p><input type="submit" value="S-Curve"/></p> +</form> + +{% endblock %}
diff --git a/lnt/server/ui/templates/v4_abtest_scurve.html b/lnt/server/ui/templates/v4_abtest_scurve.html new file mode 100644 index 0000000..85f03e2 --- /dev/null +++ b/lnt/server/ui/templates/v4_abtest_scurve.html
@@ -0,0 +1,93 @@ +{% import "utils.html" as utils %} + +{% extends "layout.html" %} +{% set components = [(ts.name, v4_url_for(".v4_recent_activity")), + ("A/B Experiments", v4_url_for(".v4_abtests")), + (exp.name or ('Experiment #' ~ exp.id), + v4_url_for(".v4_abtest", id=exp.id)), + ("S-Curve", v4_url_for(".v4_abtest_scurve", id=exp.id))] %} +{% block title %}S-Curve: {{ exp.name or exp.id }}{% endblock %} + +{% block head %} + <script src="{{ url_for('.static', filename='plotly/plotly-2.6.3.min.js') }}"></script> +{% endblock %} + +{% block body %} + +<h3> + S-Curve — {{ exp.name or '(unnamed experiment)' }} + <small>#{{ exp.id }}</small> +</h3> +<p> + <a href="{{ v4_url_for('.v4_abtest', id=exp.id) }}">← Back to experiment</a> +</p> + +{% if points %} +<div id="scurve-plot" style="width:100%; height:500px;"></div> + +<script> +(function() { + var points = {{ points | tojson }}; + + var y = points.map(function(p) { return p.pct_delta * 100; }); + var names = points.map(function(p) { return p.name; }); + + var trace = { + x: names, + y: y, + type: 'bar', + marker: { color: '#337ab7' }, + hovertemplate: '<b>%{x}</b><br>%{y:.2f}%<extra></extra>' + }; + + var zero_line = { + x: names, + y: new Array(points.length).fill(0), + mode: 'lines', + type: 'scatter', + line: { color: '#333', width: 1, dash: 'dot' }, + showlegend: false, + hoverinfo: 'skip' + }; + + var layout = { + xaxis: { + tickangle: -90 + }, + yaxis: { + title: '% change (variant vs. control)', + ticksuffix: '%', + zeroline: false + }, + hovermode: 'closest', + showlegend: false, + margin: { t: 20, r: 20, b: 400 } + }; + + Plotly.newPlot('scurve-plot', [zero_line, trace], layout, + {responsive: true, displayModeBar: true}); +})(); +</script> + +<h4>Selected benchmarks ({{ points | length }})</h4> +<table class="table table-striped table-condensed table-hover"> + <thead> + <tr><th>Rank</th><th>Benchmark</th><th>Δ</th></tr> + </thead> + <tbody> + {% for p in points %} + <tr> + <td class="right">{{ loop.index }}</td> + <td>{{ p.name }}</td> + {{ p.pct_delta|aspctcell(reverse=p.bigger_is_better)|safe }} + </tr> + {% endfor %} + </tbody> +</table> + +{% else %} +<p>No benchmarks selected. <a href="{{ v4_url_for('.v4_abtest', id=exp.id) }}">Go back</a> + and check the boxes next to the benchmarks you want to plot.</p> +{% endif %} + +{% endblock %}
diff --git a/lnt/server/ui/templates/v4_abtests.html b/lnt/server/ui/templates/v4_abtests.html new file mode 100644 index 0000000..dae9db1 --- /dev/null +++ b/lnt/server/ui/templates/v4_abtests.html
@@ -0,0 +1,35 @@ +{% import "utils.html" as utils %} + +{% extends "layout.html" %} +{% set components = [(ts.name, v4_url_for(".v4_recent_activity")), + ("A/B Experiments", v4_url_for(".v4_abtests"))] %} +{% block title %}A/B Experiments{% endblock %} + +{% block body %} +<h3>A/B Experiments</h3> + +{% if experiments %} +<table class="table table-striped table-hover table-condensed"> + <thead> + <tr> + <th>ID</th> + <th>Name</th> + <th>Created</th> + <th>Keep Forever</th> + </tr> + </thead> + <tbody> + {% for exp in experiments %} + <tr> + <td><a href="{{ v4_url_for('.v4_abtest', id=exp.id) }}">{{ exp.id }}</a></td> + <td><a href="{{ v4_url_for('.v4_abtest', id=exp.id) }}">{{ exp.name or '(unnamed)' }}</a></td> + <td>{{ exp.created_time }}</td> + <td>{{ 'yes' if exp.pinned else 'no' }}</td> + </tr> + {% endfor %} + </tbody> +</table> +{% else %} +<p>No A/B experiments yet.</p> +{% endif %} +{% endblock %}
diff --git a/lnt/server/ui/templates/v4_recent_activity.html b/lnt/server/ui/templates/v4_recent_activity.html index 3d27e1c..5112a1b 100644 --- a/lnt/server/ui/templates/v4_recent_activity.html +++ b/lnt/server/ui/templates/v4_recent_activity.html
@@ -11,6 +11,9 @@ <ul class="nav nav-list bs-docs-sidenav"> <li><a href="#machines"><i class="icon-chevron-right"></i> Active Machines</a></li> <li><a href="#submissions"><i class="icon-chevron-right"></i> Recent Submissions</a></li> + {% if recent_ab_exps %} + <li><a href="#ab_experiments"><i class="icon-chevron-right"></i> A/B Experiments</a></li> + {% endif %} </ul> {% endblock %} @@ -65,4 +68,32 @@ </tbody> </table> </section> + + {% if recent_ab_exps %} + <section id="ab_experiments"> + <h3>A/B Experiments + <small><a href="{{ v4_url_for('.v4_abtests') }}">all</a></small> + </h3> + <table class="table table-striped table-hover table-condensed"> + <thead> + <tr> + <th>ID</th> + <th>Name</th> + <th>Created</th> + <th>Pinned</th> + </tr> + </thead> + <tbody> + {% for exp in recent_ab_exps %} + <tr> + <td><a href="{{ v4_url_for('.v4_abtest', id=exp.id) }}">#{{ exp.id }}</a></td> + <td><a href="{{ v4_url_for('.v4_abtest', id=exp.id) }}">{{ exp.name or '(unnamed)' }}</a></td> + <td><span class="reltime">{{ exp.created_time.isoformat() }}</span></td> + <td>{% if exp.pinned %}<span class="label label-info">pinned</span>{% endif %}</td> + </tr> + {% endfor %} + </tbody> + </table> + </section> + {% endif %} {% endblock %}
diff --git a/lnt/server/ui/views.py b/lnt/server/ui/views.py index a8a65ee..c04cb43 100644 --- a/lnt/server/ui/views.py +++ b/lnt/server/ui/views.py
@@ -37,7 +37,7 @@ import lnt.util.stats from lnt.external.stats import stats as ext_stats from lnt.server.db import testsuitedb # noqa: F401 -from lnt.server.reporting.analysis import ComparisonResult, calc_geomean +from lnt.server.reporting.analysis import ABRunInfo, ComparisonResult, calc_geomean from lnt.server.ui import util from lnt.server.ui.decorators import frontend, db_route, v4_route from lnt.server.ui.globals import db_url_for, v4_url_for, v4_redirect @@ -261,10 +261,20 @@ active_submissions = [(r, r.order.llvm_project_revision) for r in recent_runs[:N]] + recent_ab_exps = [] + if hasattr(ts, 'ABExperiment'): + recent_ab_exps = ( + session.query(ts.ABExperiment) + .order_by(ts.ABExperiment.pinned.desc(), + ts.ABExperiment.created_time.desc()) + .limit(20) + .all()) + return render_template("v4_recent_activity.html", testsuite_name=g.testsuite_name, active_machines=active_machines, active_submissions=active_submissions, + recent_ab_exps=recent_ab_exps, **ts_data(ts)) @@ -2051,6 +2061,148 @@ **ts_data(ts)) +@v4_route("/abtests") +def v4_abtests(): + """List A/B experiments for this test suite.""" + session = request.session + ts = request.get_testsuite() + exps = session.query(ts.ABExperiment) \ + .order_by(ts.ABExperiment.created_time.desc()) \ + .limit(100).all() + data = ts_data(ts) + data['experiments'] = exps + return render_template("v4_abtests.html", **data) + + +@v4_route("/abtest/<int:id>") +def v4_abtest(id): + """Show detail and comparison results for an A/B experiment.""" + session = request.session + ts = request.get_testsuite() + + exp = session.query(ts.ABExperiment).filter_by(id=id).first() + if exp is None: + abort(404, "No A/B experiment with id %d." % id) + + control_run = session.query(ts.ABRun).filter_by( + id=exp.control_run_id).first() + variant_run = session.query(ts.ABRun).filter_by( + id=exp.variant_run_id).first() + if control_run is None or variant_run is None: + abort(404, "Experiment references a missing ABRun.") + + sri = lnt.server.reporting.analysis.ABRunInfo( + session, ts, [exp.control_run_id, exp.variant_run_id]) + test_ids = sri.test_ids + test_name_map = dict( + session.query(ts.Test.id, ts.Test.name).filter( + ts.Test.id.in_(test_ids))) + + metric_fields = [f for f in ts.sample_fields + if f.type.name in ('Real', 'Integer')] + + options = {} + options['show_all'] = bool(request.args.get('show_all')) + options['show_delta'] = bool(request.args.get('show_delta')) + options['show_small_diff'] = bool(request.args.get('show_small_diff')) + options['test_filter'] = request.args.get('test_filter', '') + + comparisons = [] + for test_id in sorted(test_ids, key=lambda tid: test_name_map.get(tid, '')): + test_name = test_name_map.get(test_id, str(test_id)) + for field in metric_fields: + cr = sri.get_run_comparison_result( + variant_run, control_run, test_id, field, None) + if cr.current is None and cr.previous is None: + continue + comparisons.append({ + 'test_name': test_name, + 'test_id': test_id, + 'field': field, + 'cr': cr, + }) + + data = ts_data(ts) + data.update({ + 'analysis': lnt.server.reporting.analysis, + 'exp': exp, + 'control_run': control_run, + 'variant_run': variant_run, + 'comparisons': comparisons, + 'metric_fields': metric_fields, + 'options': options, + }) + return render_template("v4_abtest.html", **data) + + +@v4_route("/abtest/<int:id>/pin", methods=['POST']) +def v4_abtest_pin(id): + """Toggle the pinned flag on an A/B experiment.""" + session = request.session + ts = request.get_testsuite() + exp = session.query(ts.ABExperiment).filter_by(id=id).first() + if exp is None: + abort(404, "No A/B experiment with id %d." % id) + exp.pinned = not exp.pinned + session.commit() + return v4_redirect(v4_url_for('.v4_abtest', id=id)) + + +@v4_route("/abtest/<int:id>/scurve") +def v4_abtest_scurve(id): + """S-curve graph for selected benchmarks in an A/B experiment.""" + session = request.session + ts = request.get_testsuite() + exp = session.query(ts.ABExperiment).filter_by(id=id).first() + if exp is None: + abort(404, "No A/B experiment with id %d." % id) + + control_run = session.query(ts.ABRun).filter_by( + id=exp.control_run_id).first() + variant_run = session.query(ts.ABRun).filter_by( + id=exp.variant_run_id).first() + if control_run is None or variant_run is None: + abort(404, "Experiment references a missing ABRun.") + + sri = lnt.server.reporting.analysis.ABRunInfo( + session, ts, [exp.control_run_id, exp.variant_run_id]) + test_name_map = dict( + session.query(ts.Test.id, ts.Test.name).filter( + ts.Test.id.in_(sri.test_ids))) + field_map = {f.name: f for f in ts.sample_fields + if f.type.name in ('Real', 'Integer')} + + points = [] + for param in request.args.getlist('plot'): + try: + test_id_str, field_name = param.rsplit('.', 1) + test_id = int(test_id_str) + except (ValueError, AttributeError): + continue + field = field_map.get(field_name) + if field is None: + continue + cr = sri.get_run_comparison_result( + variant_run, control_run, test_id, field, None) + if cr.pct_delta is None: + continue + points.append({ + 'name': '%s (%s)' % (test_name_map.get(test_id, str(test_id)), + field_name), + 'pct_delta': cr.pct_delta, + 'bigger_is_better': cr.bigger_is_better, + }) + + points.sort(key=lambda p: p['pct_delta']) + + data = ts_data(ts) + data.update({ + 'exp': exp, + 'points': points, + }) + return render_template("v4_abtest_scurve.html", **data) + + @frontend.route("/explode") def explode(): """This route is going to exception. Used for testing 500 page."""
diff --git a/tests/server/db/CreateV4ABTestSuite.py b/tests/server/db/CreateV4ABTestSuite.py new file mode 100644 index 0000000..ae35131 --- /dev/null +++ b/tests/server/db/CreateV4ABTestSuite.py
@@ -0,0 +1,107 @@ +# Check that ABExperiment, ABRun, and ABSample tables are created and usable. +# +# RUN: rm -f %t.db +# RUN: python %s %t.db + +import datetime + +from lnt.server.config import Config +from lnt.server.db import v4db + +# Create an in-memory database; this triggers TestSuiteDB.__init__() which +# creates all tables including the new AB tables. +db = v4db.V4DB("sqlite:///:memory:", Config.dummy_instance()) +session = db.make_session() + +# Get the test suite wrapper. +ts_db = db.testsuite['nts'] + +# Verify the new classes are accessible on the TestSuiteDB instance. +assert hasattr(ts_db, 'ABRun'), "ts_db.ABRun not found" +assert hasattr(ts_db, 'ABSample'), "ts_db.ABSample not found" +assert hasattr(ts_db, 'ABExperiment'), "ts_db.ABExperiment not found" + +# Verify table names follow the db_key_name prefix. +assert ts_db.ABRun.__tablename__ == 'NT_ABRun' +assert ts_db.ABSample.__tablename__ == 'NT_ABSample' +assert ts_db.ABExperiment.__tablename__ == 'NT_ABExperiment' + +# Create a Machine and two ABRuns for the experiment. +start_time = datetime.datetime(2024, 1, 1, 0, 0, 0) +end_time = datetime.datetime(2024, 1, 1, 0, 5, 0) + +machine = ts_db.Machine("test-machine") +machine.os = "test-os" +session.add(machine) +session.flush() + +control_run = ts_db.ABRun() +control_run.machine_id = machine.id +control_run.start_time = start_time +control_run.end_time = end_time +session.add(control_run) + +variant_run = ts_db.ABRun() +variant_run.machine_id = machine.id +variant_run.start_time = start_time +variant_run.end_time = end_time +session.add(variant_run) +session.flush() + +# Create an ABExperiment linking the two runs. +exp = ts_db.ABExperiment() +exp.name = "test-experiment" +exp.created_time = datetime.datetime.utcnow() +exp.control_run_id = control_run.id +exp.variant_run_id = variant_run.id +exp.pinned = False +session.add(exp) + +# Create a Test and ABSamples. +test = ts_db.Test("benchmark-a") +session.add(test) +session.flush() + +control_sample = ts_db.ABSample() +control_sample.run_id = control_run.id +control_sample.test_id = test.id +control_sample.compile_time = 1.0 +session.add(control_sample) + +variant_sample = ts_db.ABSample() +variant_sample.run_id = variant_run.id +variant_sample.test_id = test.id +variant_sample.compile_time = 1.05 +session.add(variant_sample) + +session.commit() + +# --- Verify round-trip --- + +exps = session.query(ts_db.ABExperiment).all() +assert len(exps) == 1, "expected 1 ABExperiment, got %d" % len(exps) +e = exps[0] +assert e.name == "test-experiment" +assert e.control_run_id == control_run.id +assert e.variant_run_id == variant_run.id +assert e.pinned is False + +ab_runs = session.query(ts_db.ABRun).all() +assert len(ab_runs) == 2, "expected 2 ABRuns, got %d" % len(ab_runs) + +ab_samples = session.query(ts_db.ABSample).all() +assert len(ab_samples) == 2, "expected 2 ABSamples, got %d" % len(ab_samples) + +# Verify dynamic metric columns were created. +assert ab_samples[0].compile_time == 1.0 +assert ab_samples[1].compile_time == 1.05 + +# Verify ABRun has no order_id (isolation guarantee). +assert not hasattr(ts_db.ABRun, 'order_id'), \ + "ABRun must not have order_idm, it must not link to the Order table" + +# Verify that item.column still points to the Sample table (not ABSample). +for field in ts_db.sample_fields: + assert field.column.table.name == ('NT_Sample'), \ + "item.column for field %r was clobbered (points to %s)" % ( + field.name, field.column.table.name)
diff --git a/tests/server/ui/test_abtest_api.py b/tests/server/ui/test_abtest_api.py new file mode 100644 index 0000000..94b35db --- /dev/null +++ b/tests/server/ui/test_abtest_api.py
@@ -0,0 +1,201 @@ +# Check the A/B testing REST API endpoints. +# RUN: rm -rf %t.instance +# RUN: python %{shared_inputs}/create_temp_instance.py \ +# RUN: %s %{shared_inputs}/SmallInstance \ +# RUN: %t.instance %S/Inputs/V4Pages_extra_records.sql +# +# RUN: python %s %t.instance + +import json +import logging +import sys +import unittest + +import lnt.server.db.migrate +import lnt.server.ui.app + +logging.basicConfig(level=logging.INFO) + +AUTH_TOKEN = 'test_token' +BASE_URL = 'api/db_default/v4/nts/' + +CONTROL_DATA = { + 'machine': {'name': 'apple-m2-macmini', 'hardware': 'arm64', 'os': 'macosx14.0'}, + 'run': { + 'start_time': '2024-01-01T00:00:00', + 'end_time': '2024-01-01T00:05:00', + }, + 'tests': [ + {'name': 'CTMark/sqlite3/sqlite3.compile', + 'compile_time': 1.0, 'execution_time': 2.0}, + {'name': 'CTMark/consumer-typeset/consumer-typeset.compile', + 'compile_time': 0.5}, + ], +} + +VARIANT_DATA = { + 'machine': {'name': 'apple-m2-macmini', 'hardware': 'arm64', 'os': 'macosx14.0'}, + 'run': { + 'start_time': '2024-01-01T00:10:00', + 'end_time': '2024-01-01T00:15:00', + }, + 'tests': [ + {'name': 'CTMark/sqlite3/sqlite3.compile', + 'compile_time': 1.05, 'execution_time': 1.95}, + {'name': 'CTMark/consumer-typeset/consumer-typeset.compile', + 'compile_time': 0.48}, + ], +} + + +class ABTestAPITest(unittest.TestCase): + def setUp(self): + _, instance_path = sys.argv + app = lnt.server.ui.app.App.create_standalone(instance_path) + app.testing = True + self.client = app.test_client() + + def _post_json(self, url, body, token=AUTH_TOKEN): + return self.client.post(url, data=json.dumps(body), + content_type='application/json', + headers={'AuthToken': token}) + + def _patch_json(self, url, body, token=AUTH_TOKEN): + return self.client.patch(url, data=json.dumps(body), + content_type='application/json', + headers={'AuthToken': token}) + + def test_01_create_experiment(self): + """POST /abtest creates an experiment and returns 201.""" + body = { + 'name': 'test-experiment', + 'control': CONTROL_DATA, + 'variant': VARIANT_DATA, + } + resp = self._post_json(BASE_URL + 'abtest', body) + self.assertEqual(resp.status_code, 201, + "Expected 201, got %d: %s" % + (resp.status_code, resp.data)) + result = json.loads(resp.data) + self.assertIn('id', result) + self.assertIn('url', result) + self.assertEqual(result['name'], 'test-experiment') + self.assertFalse(result['pinned']) + self.exp_id = result['id'] + + def test_02_get_experiment_detail(self): + """GET /abtest/<id> returns comparison results.""" + # First create an experiment. + body = { + 'name': 'detail-test', + 'control': CONTROL_DATA, + 'variant': VARIANT_DATA, + } + resp = self._post_json(BASE_URL + 'abtest', body) + self.assertEqual(resp.status_code, 201) + exp_id = json.loads(resp.data)['id'] + + # Now GET the detail. + resp = self.client.get(BASE_URL + 'abtest/%d' % exp_id) + self.assertEqual(resp.status_code, 200) + result = json.loads(resp.data) + self.assertIn('experiment', result) + self.assertIn('comparisons', result) + self.assertEqual(result['experiment']['name'], 'detail-test') + # Both tests reported compile_time; expect two compile_time comparisons. + cmp_by_field = {(c['test_name'], c['field']): c + for c in result['comparisons']} + key = ('CTMark/sqlite3/sqlite3.compile', 'compile_time') + self.assertIn(key, cmp_by_field, + "Expected sqlite3/compile_time in comparisons") + entry = cmp_by_field[key] + self.assertAlmostEqual(entry['control'], 1.0) + self.assertAlmostEqual(entry['variant'], 1.05) + + def test_03_patch_pinned(self): + """PATCH /abtest/<id> updates the pinned field.""" + body = { + 'name': 'pin-test', + 'control': CONTROL_DATA, + 'variant': VARIANT_DATA, + } + resp = self._post_json(BASE_URL + 'abtest', body) + self.assertEqual(resp.status_code, 201) + exp_id = json.loads(resp.data)['id'] + + resp = self._patch_json(BASE_URL + 'abtest/%d' % exp_id, + {'pinned': True}) + self.assertEqual(resp.status_code, 200) + result = json.loads(resp.data) + self.assertTrue(result['pinned']) + + # Verify the change is persisted. + resp = self.client.get(BASE_URL + 'abtest/%d' % exp_id) + self.assertEqual(resp.status_code, 200) + detail = json.loads(resp.data) + self.assertTrue(detail['experiment']['pinned']) + + def test_04_list_experiments(self): + """GET /abtest lists experiments.""" + body = { + 'name': 'list-test', + 'control': CONTROL_DATA, + 'variant': VARIANT_DATA, + } + self._post_json(BASE_URL + 'abtest', body) + + resp = self.client.get(BASE_URL + 'abtest') + self.assertEqual(resp.status_code, 200) + result = json.loads(resp.data) + self.assertIn('experiments', result) + self.assertGreaterEqual(len(result['experiments']), 1) + names = [e['name'] for e in result['experiments']] + self.assertIn('list-test', names) + + def test_05_list_filter_pinned(self): + """GET /abtest?pinned=true returns only pinned experiments.""" + # Create one pinned and one unpinned experiment. + for name, pinned in [('pinned-exp', True), ('unpinned-exp', False)]: + body = { + 'name': name, + 'pinned': pinned, + 'control': CONTROL_DATA, + 'variant': VARIANT_DATA, + } + resp = self._post_json(BASE_URL + 'abtest', body) + self.assertEqual(resp.status_code, 201) + + resp = self.client.get(BASE_URL + 'abtest?pinned=true') + self.assertEqual(resp.status_code, 200) + result = json.loads(resp.data) + self.assertTrue(all(e['pinned'] for e in result['experiments'])) + + def test_06_missing_control_returns_400(self): + """POST without control field returns 400.""" + body = {'name': 'bad', 'variant': VARIANT_DATA} + resp = self._post_json(BASE_URL + 'abtest', body) + self.assertEqual(resp.status_code, 400) + + def test_07_unknown_experiment_returns_404(self): + """GET /abtest/99999 returns 404 for a missing experiment.""" + resp = self.client.get(BASE_URL + 'abtest/99999') + self.assertEqual(resp.status_code, 404) + + def test_08_patch_requires_auth(self): + """PATCH without auth token returns 401.""" + body = { + 'name': 'auth-test', + 'control': CONTROL_DATA, + 'variant': VARIANT_DATA, + } + resp = self._post_json(BASE_URL + 'abtest', body) + self.assertEqual(resp.status_code, 201) + exp_id = json.loads(resp.data)['id'] + + resp = self._patch_json(BASE_URL + 'abtest/%d' % exp_id, + {'pinned': True}, token='wrong_token') + self.assertEqual(resp.status_code, 401) + + +if __name__ == '__main__': + unittest.main(argv=sys.argv[:1])
diff --git a/tests/server/ui/test_expire_abtests.py b/tests/server/ui/test_expire_abtests.py new file mode 100644 index 0000000..285a825 --- /dev/null +++ b/tests/server/ui/test_expire_abtests.py
@@ -0,0 +1,131 @@ +# Check lnt expire-abtests command. +# RUN: rm -rf %t.instance +# RUN: python %{shared_inputs}/create_temp_instance.py \ +# RUN: %s %{shared_inputs}/SmallInstance \ +# RUN: %t.instance %S/Inputs/V4Pages_extra_records.sql +# +# RUN: python %s %t.instance + +import contextlib +import datetime +import json +import sys +import unittest + +import lnt.server.db.migrate +import lnt.server.instance +import lnt.server.ui.app +from click.testing import CliRunner +from lnt.lnttool.expire_abtests import action_expire_abtests + +AUTH_TOKEN = 'test_token' +BASE_URL = 'api/db_default/v4/nts/' + +CONTROL_DATA = { + 'machine': {'name': 'apple-m2-macmini', + 'hardware': 'arm64', 'os': 'macosx14.0'}, + 'run': {'start_time': '2024-01-01T00:00:00', + 'end_time': '2024-01-01T00:05:00'}, + 'tests': [{'name': 'CTMark/sqlite3/sqlite3.compile', + 'compile_time': 1.0}], +} + +VARIANT_DATA = { + 'machine': {'name': 'apple-m2-macmini', + 'hardware': 'arm64', 'os': 'macosx14.0'}, + 'run': {'start_time': '2024-01-01T00:10:00', + 'end_time': '2024-01-01T00:15:00'}, + 'tests': [{'name': 'CTMark/sqlite3/sqlite3.compile', + 'compile_time': 1.05}], +} + + +class ExpireABTestsTest(unittest.TestCase): + def setUp(self): + _, self.instance_path = sys.argv + app = lnt.server.ui.app.App.create_standalone(self.instance_path) + app.testing = True + self.client = app.test_client() + + def _create_exp(self, name, pinned=False): + body = {'name': name, 'pinned': pinned, + 'control': CONTROL_DATA, 'variant': VARIANT_DATA} + resp = self.client.post(BASE_URL + 'abtest', + data=json.dumps(body), + content_type='application/json', + headers={'AuthToken': AUTH_TOKEN}) + self.assertEqual(resp.status_code, 201, + "POST abtest returned %d: %s" % + (resp.status_code, resp.data)) + return json.loads(resp.data)['id'] + + def _set_created_time(self, exp_id, dt): + """Back-date an experiment's created_time directly in the DB.""" + instance = lnt.server.instance.Instance.frompath(self.instance_path) + with contextlib.closing(instance.get_database('default')) as db: + session = db.make_session() + ts = db.testsuite['nts'] + exp = session.query(ts.ABExperiment).filter_by(id=exp_id).one() + exp.created_time = dt + session.commit() + + def _invoke(self, *extra_args): + runner = CliRunner() + return runner.invoke( + action_expire_abtests, + [self.instance_path, '--testsuite', 'nts'] + list(extra_args)) + + # ------------------------------------------------------------------ # + + def test_01_expire_only_old_unpinned(self): + """Only old unpinned experiments are deleted; recent and pinned survive.""" + now = datetime.datetime.utcnow() + old_id = self._create_exp('old-unpinned') + recent_id = self._create_exp('recent-unpinned') + pinned_id = self._create_exp('old-pinned', pinned=True) + self._set_created_time(old_id, now - datetime.timedelta(days=120)) + self._set_created_time(recent_id, now - datetime.timedelta(days=10)) + self._set_created_time(pinned_id, now - datetime.timedelta(days=120)) + + result = self._invoke('--older-than', '90d') + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn('Deleted 1 experiment', result.output) + self.assertIn('old-unpinned', result.output) + + # old-unpinned is gone. + self.assertEqual( + self.client.get(BASE_URL + 'abtest/%d' % old_id).status_code, 404) + # recent and pinned are still present. + self.assertEqual( + self.client.get(BASE_URL + 'abtest/%d' % recent_id).status_code, 200) + self.assertEqual( + self.client.get(BASE_URL + 'abtest/%d' % pinned_id).status_code, 200) + + def test_02_dry_run_leaves_experiments_intact(self): + """--dry-run reports what would be deleted but changes nothing.""" + now = datetime.datetime.utcnow() + exp_id = self._create_exp('dry-run-target') + self._set_created_time(exp_id, now - datetime.timedelta(days=200)) + + result = self._invoke('--older-than', '90d', '--dry-run') + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn('Would delete', result.output) + self.assertIn('dry-run-target', result.output) + # Experiment still exists. + self.assertEqual( + self.client.get(BASE_URL + 'abtest/%d' % exp_id).status_code, 200) + + def test_03_nothing_to_delete(self): + """When nothing qualifies, a suitable message is printed.""" + result = self._invoke('--older-than', '1y') + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn('No experiments to delete', result.output) + + def test_04_bad_age_format_exits_nonzero(self): + """An unrecognised age string causes a non-zero exit.""" + result = self._invoke('--older-than', 'banana') + self.assertNotEqual(result.exit_code, 0) + + +if __name__ == '__main__': + unittest.main(argv=sys.argv[:1])
diff --git a/tests/server/ui/v4/V4Pages.py b/tests/server/ui/v4/V4Pages.py index 58288d0..e62ec74 100644 --- a/tests/server/ui/v4/V4Pages.py +++ b/tests/server/ui/v4/V4Pages.py
@@ -788,6 +788,40 @@ check_html(client, f'/v4/nts/{profile_last_run["id"]}') check_html(client, f'/v4/nts/{failed_run["id"]}') + # A/B experiment list page (empty is fine). + check_html(client, '/db_default/v4/nts/abtests') + + # Create an A/B experiment via the API and then check both web pages. + ab_body = json.dumps({ + 'name': 'v4pages-smoke', + 'control': { + 'machine': {'name': 'apple-m2-macmini', 'hardware': 'arm64', + 'os': 'macosx14.0'}, + 'run': {'start_time': '2024-01-01T00:00:00', + 'end_time': '2024-01-01T00:05:00'}, + 'tests': [{'name': 'CTMark/sqlite3/sqlite3.compile', + 'compile_time': 1.0}], + }, + 'variant': { + 'machine': {'name': 'apple-m2-macmini', 'hardware': 'arm64', + 'os': 'macosx14.0'}, + 'run': {'start_time': '2024-01-01T00:10:00', + 'end_time': '2024-01-01T00:15:00'}, + 'tests': [{'name': 'CTMark/sqlite3/sqlite3.compile', + 'compile_time': 1.05}], + }, + }) + ab_resp = client.post('api/db_default/v4/nts/abtest', + data=ab_body, content_type='application/json', + headers={'AuthToken': 'test_token'}) + assert ab_resp.status_code == 201, \ + "AB experiment POST returned %d" % ab_resp.status_code + ab_id = json.loads(ab_resp.data)['id'] + check_html(client, '/db_default/v4/nts/abtests') + check_html(client, '/db_default/v4/nts/abtest/%d' % ab_id) + check_code(client, '/db_default/v4/nts/abtest/99999', + expected_code=HTTP_NOT_FOUND) + if __name__ == '__main__': main()