Display additional run information on data points on charts (#148)
This allows arbitrary run information to be displayed on the charts
when clicking the data point. The content of the run information is
controled by the submitter of that run, which allows a great deal of
flexibility.
Folks will want to avoid attaching too much information to the runs
to avoid making graphs too slow, but that's on them.
Fixes #75
diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst
index 1cdfac5..46feeb3 100644
--- a/docs/developer_guide.rst
+++ b/docs/developer_guide.rst
@@ -19,7 +19,8 @@
pip install ".[dev]"
This will install the current version of the package, along with the dependencies
-required for development (``lit``, ``filecheck``, etc).
+required for development (``lit``, ``filecheck``, etc). Note that ``curl`` and
+``jq`` are also required for running the tests.
Running LNT's Regression Tests
------------------------------
diff --git a/docs/importing_data.rst b/docs/importing_data.rst
index 2d8d58a..2168d80 100644
--- a/docs/importing_data.rst
+++ b/docs/importing_data.rst
@@ -58,16 +58,23 @@
]
}
+Any optional fields provided in the ``run`` section will be associated to that run and visible
+in the UI when looking at that run. This allows annotating runs with useful additional information,
+such as the commit information related to this run or similar. This data will also be visible on
+charts when viewing historical results. Arbitrary data can be provided, however including too much
+run-related information can cause very large amounts of data to have to be transfered to display
+graphs.
+
A concrete small example is
.. literalinclude:: report-example.json
:language: json
-Given how simple it is to make your own results and send them to LNT,
-it is common to not use the LNT client application at all, and just have a
-custom script run your tests and submit the data to the LNT server. Details
-on how to do this are in :mod:`lnt.testing`
+Given how simple it is to make your own results and send them to LNT, it is common to
+not use the LNT client application at all, and just have a custom script run your tests
+and submit the data to the LNT server in JSON format.
+
.. _nts_suite:
diff --git a/lnt/server/ui/static/lnt_graph.js b/lnt/server/ui/static/lnt_graph.js
index 6737b65..01ecb65 100644
--- a/lnt/server/ui/static/lnt_graph.js
+++ b/lnt/server/ui/static/lnt_graph.js
@@ -101,7 +101,15 @@
get_run_url(db_name, test_suite_name, point.meta.runID) +
"\">" + point.meta.runID + "</a><br>";
}
-
+
+ // Display any additional run parameters
+ var known_fields = ['order', 'orderID', 'date', 'runID', 'state'];
+ for (var key in point.meta) {
+ if (point.meta.hasOwnProperty(key) && known_fields.indexOf(key) === -1) {
+ tip_body += "<b>" + key + ":</b> " + point.meta[key] + "<br>";
+ }
+ }
+
if (point.meta.runID && point.data.url) { // url = machine.id/test.id/field_index
tip_body += "<a href=\"" +
get_manual_regression_url(db_name, test_suite_name, point.data.url, point.meta.runID) +
diff --git a/lnt/server/ui/views.py b/lnt/server/ui/views.py
index 7213c4c..4695bed 100644
--- a/lnt/server/ui/views.py
+++ b/lnt/server/ui/views.py
@@ -899,7 +899,7 @@
# FIXME: Don't join to Order here, aggregate this across all the tests
# we want to load. Actually, we should just make this a single query.
values = session.query(plot_parameter.field.column, ts.Order,
- ts.Run.start_time, ts.Run.id) \
+ ts.Run.start_time, ts.Run.id, ts.Run.parameters_data) \
.select_from(ts.Sample) \
.join(ts.Run).join(ts.Order) \
.filter(ts.Run.machine_id == plot_parameter.machine.id) \
@@ -916,15 +916,15 @@
if xaxis_date:
# Aggregate by date.
data = list(multidict.multidict(
- (date, (val, order, date, run_id))
- for val, order, date, run_id in values).items())
+ (date, (val, order, date, run_id, parameters_data))
+ for val, order, date, run_id, parameters_data in values).items())
# Sort data points according to date.
data.sort(key=lambda sample: sample[0])
else:
# Aggregate by order (revision).
data = list(multidict.multidict(
- (order.llvm_project_revision, (val, order, date, run_id))
- for val, order, date, run_id in values).items())
+ (order.llvm_project_revision, (val, order, date, run_id, parameters_data))
+ for val, order, date, run_id, parameters_data in values).items())
# Sort data points according to order (revision).
data.sort(key=lambda sample: convert_revision(sample[0], cache=revision_cache))
@@ -1194,7 +1194,9 @@
# And the date on which they were taken.
dates = [data_array[2] for data_array in datapoints]
# Run ID where this point was collected.
- run_ids = [data_array[3] for data_array in datapoints if len(data_array) == 4]
+ run_ids = [data_array[3] for data_array in datapoints if len(data_array) >= 4]
+ # Run parameters (JSON-encoded binary data)
+ run_parameters_list = [data_array[4] for data_array in datapoints if len(data_array) >= 5]
values = [v * normalize_by for v in values]
@@ -1224,6 +1226,13 @@
"date": str(dates[agg_index])}
if run_ids:
point_metadata["runID"] = str(run_ids[agg_index])
+
+ # Add run parameters to metadata if available
+ if run_parameters_list:
+ run_params = run_parameters_list[agg_index]
+ if run_params:
+ point_metadata.update(json.loads(run_params.decode('utf-8')))
+
meta.append(point_metadata)
# Add the multisample points, if requested.
@@ -1235,6 +1244,13 @@
"date": str(dates[i])}
if run_ids:
multisample_metadata["runID"] = str(run_ids[i])
+
+ # Add run parameters to metadata if available
+ if run_parameters_list:
+ run_params = run_parameters_list[i]
+ if run_params:
+ multisample_metadata.update(json.loads(run_params.decode('utf-8')))
+
multisample_points_data["x"].append(point_label)
multisample_points_data["y"].append(v)
multisample_points_data["meta"].append(multisample_metadata)
diff --git a/tests/lnttool/submit.extra-run-information.shtest b/tests/lnttool/submit.extra-run-information.shtest
new file mode 100644
index 0000000..5b1bb63
--- /dev/null
+++ b/tests/lnttool/submit.extra-run-information.shtest
@@ -0,0 +1,70 @@
+#!/bin/bash
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: lnt create %t/instance
+# RUN: %{shared_inputs}/server_wrapper.sh --fail-on-error %t/instance 9094 \
+# RUN: /bin/sh %s %t
+
+#
+# Make sure that we can have extra information in the run, and that such information
+# is saved into the DB.
+#
+
+set -eux
+
+t="$1"
+HOST="http://localhost:9094"
+
+# Simple case
+cat <<EOF > "$t/report1.json"
+{
+ "format_version": "2",
+ "machine": {
+ "name": "machine-foo"
+ },
+ "run": {
+ "start_time": "2026-01-01T11:28:33.00000",
+ "end_time": "2026-01-01T11:28:23.991076",
+ "llvm_project_revision": "1",
+ "some_extra_information1": "some tokens",
+ "some_extra_information2": "more tokens"
+ },
+ "tests": [
+ {
+ "name": "benchmark1",
+ "execution_time": [ 0.1056, 0.1055 ]
+ }
+ ]
+}
+EOF
+
+lnt submit "${HOST}/db_default/v4/nts/submitRun" "$t/report1.json"
+curl -sS "${HOST}/api/db_default/v4/nts/runs/1" > "$t/run1.json"
+jq .run.some_extra_information1 "$t/run1.json" | grep "some tokens"
+jq .run.some_extra_information2 "$t/run1.json" | grep "more tokens"
+
+
+# Add different fields with another run
+cat <<EOF > "$t/report2.json"
+{
+ "format_version": "2",
+ "machine": {
+ "name": "machine-foo"
+ },
+ "run": {
+ "start_time": "2026-01-02T11:28:33.00000",
+ "end_time": "2026-01-02T11:28:23.991076",
+ "llvm_project_revision": "2",
+ "some_extra_information3": "some tokens 3"
+ },
+ "tests": [
+ {
+ "name": "benchmark1",
+ "execution_time": [ 0.1056, 0.1055 ]
+ }
+ ]
+}
+EOF
+
+lnt submit "${HOST}/db_default/v4/nts/submitRun" "$t/report2.json"
+curl -sS "${HOST}/api/db_default/v4/nts/runs/2" > "$t/run2.json"
+jq .run.some_extra_information3 "$t/run2.json" | grep "some tokens 3"
