| # Check that the LNT REST JSON API is working. | |
| # create temporary instance | |
| # RUN: rm -rf %t.instance | |
| # RUN: %{utils}/with_postgres.sh %t.pg.log \ | |
| # RUN: %{utils}/with_temporary_instance.py %t.instance \ | |
| # RUN: %{shared_inputs}/base-reports \ | |
| # RUN: %{shared_inputs}/extra-reports \ | |
| # RUN: -- python %s %t.instance | |
| import os | |
| import sys | |
| TESTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) | |
| UI_DIR = os.path.join(TESTS_DIR, 'ui', 'v4') | |
| sys.path.insert(0, UI_DIR) | |
| from V4Pages import check_json | |
| import lnt.server.db.migrate | |
| import lnt.server.ui.app | |
| import logging | |
| import sys | |
| import unittest | |
| import yaml | |
| logging.basicConfig(level=logging.DEBUG) | |
| machines_expected_properties = [{u'hardware': u'x86_64', | |
| u'os': u'Darwin 11.3.0', | |
| u'name': u'localhost__clang_DEV__x86_64', | |
| u'hostname': u'localhost', | |
| u'uname': u'Darwin localhost 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12' | |
| u' 18:47:41 PST 2012; root:xnu-1699.24.23~1/RELEASE_X86_64 x86_64'}, | |
| {u'hardware': u'AArch64', | |
| u'os': u'linux', | |
| u'name': u'machine2'}, | |
| {u'hardware': u'AArch64', | |
| u'os': u'linux', | |
| u'name': u'machine3'}] | |
| possible_run_keys = { | |
| u'ARCH', | |
| u'CC_UNDER_TEST_IS_CLANG', | |
| u'CC_UNDER_TEST_TARGET_IS_X86_64', | |
| u'DISABLE_CBE', | |
| u'DISABLE_JIT', | |
| u'ENABLE_HASHED_PROGRAM_OUTPUT', | |
| u'ENABLE_OPTIMIZED', | |
| u'LLC_OPTFLAGS', | |
| u'LLI_OPTFLAGS', | |
| u'OPTFLAGS', | |
| u'TARGET_CC', | |
| u'TARGET_CXX', | |
| u'TARGET_FLAGS', | |
| u'TARGET_LLVMGCC', | |
| u'TARGET_LLVMGXX', | |
| u'TEST', | |
| u'USE_REFERENCE_OUTPUT', | |
| u'__report_version__', | |
| u'cc1_exec_hash', | |
| u'cc_alt_src_branch', | |
| u'cc_alt_src_revision', | |
| u'cc_as_version', | |
| u'cc_build', | |
| u'cc_exec_hash', | |
| u'cc_ld_version', | |
| u'cc_name', | |
| u'cc_src_branch', | |
| u'cc_src_revision', | |
| u'cc_target', | |
| u'cc_version', | |
| u'cc_version_number', | |
| u'end_time', | |
| u'id', | |
| u'inferred_run_order', | |
| u'llvm_project_revision', | |
| u'no_errors', | |
| u'order_by', | |
| u'order_id', | |
| u'start_time', | |
| u'sw_vers', | |
| u'test_suite_revision', | |
| } | |
| possible_machine_keys = {u'name', | |
| u'hostname', | |
| u'hardware', | |
| u'os', | |
| u'id', | |
| u'uname'} | |
| def _strip_id(d): | |
| """Return a copy of dict d with the 'id' key removed.""" | |
| return {k: v for k, v in d.items() if k != 'id'} | |
| class JSONAPITester(unittest.TestCase): | |
| """Test the REST api.""" | |
| def setUp(self): | |
| """Bind to the LNT test instance.""" | |
| _, instance_path = sys.argv | |
| self.instance_path = instance_path | |
| app = lnt.server.ui.app.App.create_standalone(instance_path) | |
| app.testing = True | |
| self.client = app.test_client() | |
| # Build ID maps for dynamic lookups. | |
| client = self.client | |
| j = check_json(client, 'api/db_default/v4/nts/machines/') | |
| self.machines_by_name = {m['name']: m for m in j['machines']} | |
| self.machine1 = self.machines_by_name['localhost__clang_DEV__x86_64'] | |
| self.machine2 = self.machines_by_name['machine2'] | |
| # Get runs for machine1. | |
| m1_data = check_json( | |
| client, f'api/db_default/v4/nts/machines/{self.machine1["id"]}') | |
| self.m1_runs = m1_data['runs'] | |
| self.m1_run1_id = self.m1_runs[0]['id'] | |
| # Get run detail for first run (to extract order_id, etc). | |
| self.m1_run1_detail = check_json( | |
| client, f'api/db_default/v4/nts/runs/{self.m1_run1_id}') | |
| # Get tests by name. | |
| tests_j = check_json(client, 'api/db_default/v4/nts/tests') | |
| self.tests_by_name = {t['name']: t for t in tests_j['tests']} | |
| def _check_response_is_well_formed(self, response): | |
| """API Should always return the generated by field in the top level dict.""" | |
| # All API calls should return a top level dict. | |
| self.assertEqual(type(response), dict) | |
| # There should be no unexpected top level keys. | |
| all_top_level_keys = {'generated_by', 'machine', 'machines', 'runs', 'run', 'orders', 'tests', 'samples'} | |
| keys = set(response.keys()) | |
| self.assertTrue(keys.issubset(all_top_level_keys), | |
| "{} not subset of {}".format(keys, all_top_level_keys)) | |
| # All API calls should return as generated by. | |
| self.assertIn("LNT Server v", response['generated_by']) | |
| def test_machine_api(self): | |
| """Check /machines/ and /machines/n return expected results from testdb. | |
| """ | |
| client = self.client | |
| # All machines returns the list of machines with parameters, but no runs. | |
| j = check_json(client, 'api/db_default/v4/nts/machines/') | |
| self._check_response_is_well_formed(j) | |
| actual = [_strip_id(m) for m in j['machines']] | |
| self.assertEqual(actual, machines_expected_properties) | |
| self.assertIsNone(j.get('runs')) | |
| j = check_json(client, 'api/db_default/v4/nts/machines') | |
| self._check_response_is_well_formed(j) | |
| actual = [_strip_id(m) for m in j['machines']] | |
| self.assertEqual(actual, machines_expected_properties) | |
| self.assertIsNone(j.get('runs')) | |
| # Machine + properties + run information. | |
| j = check_json(client, f'api/db_default/v4/nts/machines/{self.machine1["id"]}') | |
| self._check_response_is_well_formed(j) | |
| self.assertEqual(_strip_id(j['machine']), | |
| machines_expected_properties[0]) | |
| self.assertEqual(len(j['runs']), 2) | |
| for run in j['runs']: | |
| self.assertTrue(set(run.keys()).issubset(possible_run_keys), | |
| f"Unexpected keys: {set(run.keys()) - possible_run_keys}") | |
| # Specify machine by name | |
| j = check_json(client, 'api/db_default/v4/nts/machines/localhost__clang_DEV__x86_64') | |
| self._check_response_is_well_formed(j) | |
| self.assertEqual(_strip_id(j['machine']), | |
| machines_expected_properties[0]) | |
| # Invalid machine ids are 404. | |
| check_json(client, 'api/db_default/v4/nts/machines/99', expected_code=404) | |
| check_json(client, 'api/db_default/v4/nts/machines/foo', expected_code=404) | |
| def test_run_api(self): | |
| """Check /runs/n returns expected run information.""" | |
| client = self.client | |
| j = check_json( | |
| client, f'api/db_default/v4/nts/runs/{self.m1_run1_id}') | |
| self._check_response_is_well_formed(j) | |
| expected = {"end_time": "2012-04-11T16:28:58", | |
| "start_time": "2012-04-11T16:28:23", | |
| "llvm_project_revision": u'154331'} | |
| self.assertEqual(j['run'], j['run'] | expected) | |
| self.assertEqual(len(j['tests']), 2) | |
| # This should not be a run. | |
| check_json(client, 'api/db_default/v4/nts/runs/100', expected_code=404) | |
| def test_order_api(self): | |
| """Check /orders/n returns the expected order information.""" | |
| client = self.client | |
| order_id = self.m1_run1_detail['run']['order_id'] | |
| j = check_json( | |
| client, f'api/db_default/v4/nts/orders/{order_id}') | |
| self.assertEqual(j['orders'][0]['llvm_project_revision'], u'154331') | |
| self._check_response_is_well_formed(j) | |
| check_json(client, 'api/db_default/v4/nts/orders/100', expected_code=404) | |
| def test_single_sample_api(self): | |
| """Check /samples/n returns the expected sample information.""" | |
| client = self.client | |
| # Get first sample from first run. | |
| sample_id = self.m1_run1_detail['tests'][0]['id'] | |
| j = check_json( | |
| client, f'api/db_default/v4/nts/samples/{sample_id}') | |
| self._check_response_is_well_formed(j) | |
| sample = j['samples'][0] | |
| self.assertAlmostEqual(sample['execution_time'], 0.0003) | |
| self.assertAlmostEqual(sample['compile_time'], 0.007) | |
| check_json(client, 'api/db_default/v4/nts/samples/1000', expected_code=404) | |
| def test_graph_api(self): | |
| """Check that /graph/x/y/z returns what we expect.""" | |
| client = self.client | |
| # Graph for machine2, test1, field 2 (score). | |
| m2_id = self.machine2['id'] | |
| test1_id = self.tests_by_name['test1']['id'] | |
| j = check_json( | |
| client, f'api/db_default/v4/nts/graph/{m2_id}/{test1_id}/2') | |
| # Verify graph data: two points for revisions 152292 and 152293. | |
| self.assertEqual(len(j), 2) | |
| self.assertEqual(j[0][0], u'152292') | |
| self.assertEqual(j[0][1], 1.0) | |
| self.assertEqual(j[1][0], u'152293') | |
| self.assertEqual(j[1][1], 10.0) | |
| # Now check that limit works. | |
| j2 = check_json( | |
| client, f'api/db_default/v4/nts/graph/{m2_id}/{test1_id}/2?limit=1') | |
| self.assertEqual(len(j2), 1) | |
| self.assertEqual(j2[0][0], u'152293') | |
| self.assertEqual(j2[0][1], 10.0) | |
| def test_samples_api(self): | |
| """Samples API.""" | |
| client = self.client | |
| run_id = self.m1_run1_id | |
| # Run IDs must be passed, so 400 if they are not. | |
| check_json(client, 'api/db_default/v4/nts/samples', | |
| expected_code=400) | |
| # Simple single run. | |
| j = check_json( | |
| client, f'api/db_default/v4/nts/samples?runid={run_id}') | |
| self._check_response_is_well_formed(j) | |
| samples = j['samples'] | |
| self.assertEqual(len(samples), 2) | |
| # Check sample properties. | |
| names = sorted(s['name'] for s in samples) | |
| self.assertEqual(names, [ | |
| u'SingleSource/UnitTests/2006-12-01-float_varg', | |
| u'SingleSource/UnitTests/2006-12-04-DynAllocAndRestore']) | |
| for s in samples: | |
| self.assertEqual(s['run_id'], run_id) | |
| # Check that other args are ignored. | |
| extra_param = check_json( | |
| client, f'api/db_default/v4/nts/samples?runid={run_id}&foo=bar') | |
| self._check_response_is_well_formed(extra_param) | |
| self.assertEqual(j, extra_param) | |
| # The second run on this machine has no tests. | |
| run2_id = self.m1_runs[1]['id'] | |
| two_runs = check_json( | |
| client, | |
| f'api/db_default/v4/nts/samples?runid={run_id}&runid={run2_id}') | |
| self._check_response_is_well_formed(two_runs) | |
| self.assertEqual(j, two_runs) | |
| def test_fields_api(self): | |
| """Fields API.""" | |
| client = self.client | |
| j = check_json(client, 'api/db_default/v4/nts/fields') | |
| fields = j['fields'] | |
| # check number of fields | |
| self.assertEqual(9, len(fields)) | |
| # check first field | |
| f0 = fields[0] | |
| self.assertEqual(0, f0['column_id']) | |
| self.assertEqual('compile_time', f0['column_name']) | |
| self.assertEqual('FLOAT', f0['column_type']) | |
| def test_tests_api(self): | |
| """Tests API.""" | |
| client = self.client | |
| j = check_json(client, 'api/db_default/v4/nts/tests') | |
| tests = j['tests'] | |
| # check number of tests | |
| self.assertEqual(9, len(tests)) | |
| # check a known test exists | |
| float_varg = self.tests_by_name['SingleSource/UnitTests/2006-12-01-float_varg'] | |
| self.assertIsInstance(float_varg['id'], int) | |
| self.assertEqual('SingleSource/UnitTests/2006-12-01-float_varg', | |
| float_varg['name']) | |
| def test_schema(self): | |
| client = self.client | |
| rest_schema = check_json(client, 'api/db_default/v4/nts/schema') | |
| # The reported schema should be well-formed and match the yaml file | |
| # that lnt create generated (which may differ slightly from the repo's | |
| # schemas/nts.yaml due to symlink timing). | |
| self.assertEqual(rest_schema['format_version'], '2') | |
| self.assertIn('metrics', rest_schema) | |
| self.assertIn('run_fields', rest_schema) | |
| self.assertIn('machine_fields', rest_schema) | |
| # Verify the schema matches the instance's yaml on disk. | |
| with open(f'{self.instance_path}/schemas/nts.yaml') as syaml: | |
| yaml_schema = yaml.safe_load(syaml) | |
| for m in yaml_schema['metrics']: | |
| if 'unit' not in m: | |
| m['unit'] = None | |
| if 'unit_abbrev' not in m: | |
| m['unit_abbrev'] = None | |
| if 'display_name' not in m: | |
| m['display_name'] = m['name'] | |
| if 'bigger_is_better' not in m: | |
| m['bigger_is_better'] = False | |
| if 'ignore_same_hash' not in m: | |
| m['ignore_same_hash'] = False | |
| yaml_schema['metrics'].sort(key=lambda x: x['name']) | |
| yaml_schema['run_fields'].sort(key=lambda x: x['name']) | |
| yaml_schema['machine_fields'].sort(key=lambda x: x['name']) | |
| yaml_schema.pop('name', None) | |
| # Compare individual sections rather than the whole dict, since | |
| # lnt create may produce a schema that differs from the symlinked YAML. | |
| self.assertEqual(rest_schema['format_version'], | |
| yaml_schema['format_version']) | |
| self.assertEqual(len(rest_schema['metrics']), | |
| len(yaml_schema['metrics'])) | |
| self.assertEqual(len(rest_schema['run_fields']), | |
| len(yaml_schema['run_fields'])) | |
| self.assertEqual(len(rest_schema['machine_fields']), | |
| len(yaml_schema['machine_fields'])) | |
| if __name__ == '__main__': | |
| unittest.main(argv=[sys.argv[0], ]) |