fix lint error Created using spr 1.3.7
diff --git a/lnt/lnttool/__init__.py b/lnt/lnttool/__init__.py index a949161..014ea31 100644 --- a/lnt/lnttool/__init__.py +++ b/lnt/lnttool/__init__.py
@@ -13,6 +13,7 @@ from .runtest import group_runtest from .send_daily_report import action_send_daily_report from .send_run_comparison import action_send_run_comparison +from .abtest import group_abtest from .expire_abtests import action_expire_abtests from .showtests import action_showtests from .submit import action_submit @@ -46,6 +47,7 @@ main.add_command(action_checkformat) main.add_command(action_convert) main.add_command(action_create) +main.add_command(group_abtest) main.add_command(action_expire_abtests) main.add_command(action_import) main.add_command(action_importreport)
diff --git a/lnt/lnttool/abtest.py b/lnt/lnttool/abtest.py new file mode 100644 index 0000000..99488f5 --- /dev/null +++ b/lnt/lnttool/abtest.py
@@ -0,0 +1,141 @@ +"""lnt abtest — manage A/B performance experiments on a remote LNT server.""" +import json +import ssl +import urllib.error +import urllib.request + +import certifi +import click + + +def _api_url(server_url, database, testsuite, *path_parts): + base = '%s/api/db_%s/v4/%s' % (server_url.rstrip('/'), database, testsuite) + if path_parts: + return '%s/%s' % (base, '/'.join(str(p) for p in path_parts)) + return base + + +def _api_request(method, url, body=None, auth_token=None): + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + if auth_token: + headers['AuthToken'] = auth_token + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, headers=headers, method=method) + context = ssl.create_default_context(cafile=certifi.where()) + try: + resp = urllib.request.urlopen(req, context=context) + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + body_text = e.read().decode(errors='replace') + raise click.ClickException('HTTP %d: %s' % (e.code, body_text)) + except urllib.error.URLError as e: + raise click.ClickException('Could not connect to %s: %s' % (url, e)) + + +@click.group("abtest") +def group_abtest(): + """manage A/B performance experiments on a remote LNT server""" + + +@group_abtest.command("create") +@click.argument("server_url") +@click.option("--database", default="default", show_default=True, + help="LNT database name") +@click.option("--testsuite", "-s", default="nts", show_default=True, + help="testsuite name") +@click.option("--name", default="", + help="human-readable experiment name") +@click.option("--control", "control_file", + type=click.Path(exists=True), default=None, + help="control run report JSON") +@click.option("--variant", "variant_file", + type=click.Path(exists=True), default=None, + help="variant run report JSON") +@click.option("--auth-token", envvar="LNT_AUTH_TOKEN", + help="API auth token (or set LNT_AUTH_TOKEN)") +def action_abtest_create(server_url, database, testsuite, name, + control_file, variant_file, auth_token): + """Create an A/B experiment on a remote LNT server. + +\b +Two workflows are supported: + + Atomic — both runs available at the same time: + + lnt abtest create SERVER --name "pr-42" \\ + --control control.json --variant variant.json + + Two-phase — control and variant submitted by independent CI jobs: + + # Orchestrator: create the experiment and capture the ID + ID=$(lnt abtest create SERVER --name "pr-42") + + # Control CI job + lnt abtest submit SERVER $ID --control control.json + + # Variant CI job + lnt abtest submit SERVER $ID --variant variant.json + """ + if bool(control_file) != bool(variant_file): + raise click.UsageError( + "Provide both --control and --variant for atomic creation, " + "or neither to create a pending experiment.") + + body = {'name': name} + if control_file: + with open(control_file) as f: + body['control'] = json.load(f) + with open(variant_file) as f: + body['variant'] = json.load(f) + + url = _api_url(server_url, database, testsuite, 'abtest') + result = _api_request('POST', url, body=body, auth_token=auth_token) + + # Print just the ID to stdout so scripts can capture it with $(...). + click.echo(result['id']) + exp_url = result.get('url') + if exp_url: + click.echo('Experiment: %s' % exp_url, err=True) + + +@group_abtest.command("submit") +@click.argument("server_url") +@click.argument("experiment_id", type=int) +@click.option("--database", default="default", show_default=True, + help="LNT database name") +@click.option("--testsuite", "-s", default="nts", show_default=True, + help="testsuite name") +@click.option("--control", "control_file", + type=click.Path(exists=True), default=None, + help="submit this JSON as the control run") +@click.option("--variant", "variant_file", + type=click.Path(exists=True), default=None, + help="submit this JSON as the variant run") +@click.option("--auth-token", envvar="LNT_AUTH_TOKEN", + help="API auth token (or set LNT_AUTH_TOKEN)") +def action_abtest_submit(server_url, experiment_id, database, testsuite, + control_file, variant_file, auth_token): + """Submit a control or variant run to an existing A/B experiment. + +\b +Used in the two-phase workflow after 'lnt abtest create' has returned an ID: + + lnt abtest submit SERVER ID --control control.json + lnt abtest submit SERVER ID --variant variant.json + """ + if not control_file and not variant_file: + raise click.UsageError("Provide --control or --variant.") + if control_file and variant_file: + raise click.UsageError( + "Provide --control or --variant, not both. " + "To submit both at once use 'lnt abtest create'.") + + role = 'control' if control_file else 'variant' + report_file = control_file or variant_file + + with open(report_file) as f: + body = json.load(f) + + url = _api_url(server_url, database, testsuite, 'abtest', experiment_id, role) + _api_request('POST', url, body=body, auth_token=auth_token) + click.echo('Submitted %s run for experiment %d.' % (role, experiment_id))
diff --git a/lnt/server/ui/api.py b/lnt/server/ui/api.py index b1998ec..0b20415 100644 --- a/lnt/server/ui/api.py +++ b/lnt/server/ui/api.py
@@ -609,24 +609,39 @@ @staticmethod @requires_auth_token def post(): - """Submit control and variant reports and create an ABExperiment.""" + """Create an ABExperiment, optionally with both runs at once. + + Atomic mode: provide both 'control' and 'variant' keys in the body. + Two-phase mode: omit both; submit runs later via + POST /abtest/<id>/control and POST /abtest/<id>/variant. + """ session = request.session ts = request.get_testsuite() body = request.get_json(force=True) if body is None: abort(400, msg="Request body must be JSON.") - for key in ('control', 'variant'): - if key not in body: - abort(400, msg="Missing required field: %r" % key) - control_run = ts.importABDataFromDict(session, body['control']) - session.flush() - variant_run = ts.importABDataFromDict(session, body['variant']) - session.flush() + + has_control = 'control' in body + has_variant = 'variant' in body + if has_control != has_variant: + abort(400, msg="Provide both 'control' and 'variant' for atomic " + "creation, or neither to create a pending experiment.") + + control_run_id = None + variant_run_id = None + if has_control: + control_run = ts.importABDataFromDict(session, body['control']) + session.flush() + variant_run = ts.importABDataFromDict(session, body['variant']) + session.flush() + control_run_id = control_run.id + variant_run_id = variant_run.id + exp = ts.ABExperiment() exp.name = body.get('name', '') exp.created_time = datetime.datetime.utcnow() - exp.control_run_id = control_run.id - exp.variant_run_id = variant_run.id + exp.control_run_id = control_run_id + exp.variant_run_id = variant_run_id exp.pinned = bool(body.get('pinned', False)) session.add(exp) session.commit() @@ -655,19 +670,59 @@ return {'experiments': [_ab_exp_to_dict(e) for e in exps]} +class ABTestRun(Resource): + """Submit a single run (control or variant) to an existing experiment.""" + method_decorators = [in_db] + + @staticmethod + @requires_auth_token + def post(abtest_id, role): + """Import a run and attach it to an existing experiment as + 'control' or 'variant'. The request body is a standard LNT report + JSON object (same format accepted by POST /runs).""" + if role not in ('control', 'variant'): + abort(400, msg="role must be 'control' or 'variant'") + session = request.session + ts = request.get_testsuite() + exp = session.query(ts.ABExperiment).filter_by(id=abtest_id).first() + if exp is None: + abort(404, msg="No such A/B experiment.") + body = request.get_json(force=True) + if body is None: + abort(400, msg="Request body must be JSON.") + ab_run = ts.importABDataFromDict(session, body) + session.flush() + if role == 'control': + exp.control_run_id = ab_run.id + else: + exp.variant_run_id = ab_run.id + session.commit() + return _ab_exp_to_dict(exp) + + class ABTestDetail(Resource): """Retrieve or update a single A/B experiment.""" method_decorators = [in_db] @staticmethod def get(abtest_id): - """Return the experiment metadata and per-test comparison results.""" + """Return the experiment metadata and per-test comparison results. + + If either run has not been submitted yet the response includes + ``"pending": true`` and an empty ``comparisons`` list.""" session = request.session ts = request.get_testsuite() exp = session.query(ts.ABExperiment).filter_by(id=abtest_id).first() if exp is None: abort(404, msg="No such A/B experiment.") + # Pending state: one or both runs not yet submitted. + if exp.control_run_id is None or exp.variant_run_id is None: + result = _ab_exp_to_dict(exp) + result['pending'] = True + result['comparisons'] = [] + return result + control_run = session.query(ts.ABRun).filter_by( id=exp.control_run_id).first() variant_run = session.query(ts.ABRun).filter_by( @@ -760,3 +815,5 @@ api.add_resource(Regression, ts_path(regression_url)) api.add_resource(ABTests, ts_path("abtest"), ts_path("abtest/")) api.add_resource(ABTestDetail, ts_path("abtest/<int:abtest_id>")) + api.add_resource(ABTestRun, + ts_path("abtest/<int:abtest_id>/<string:role>"))