)]}'
{
  "commit": "3fbb3ac596f5e1b5376155003361282cdf7279df",
  "tree": "723fbcff72cbbc31b3aded019fbb83e3de04d6e8",
  "parents": [
    "91b9dd15d86f3541a7fb4fc4b59cea1da919b3d6"
  ],
  "author": {
    "name": "Henry Jiang",
    "email": "henry_jiang2@apple.com",
    "time": "Thu Apr 09 09:49:03 2026 -0700"
  },
  "committer": {
    "name": "GitHub",
    "email": "noreply@github.com",
    "time": "Thu Apr 09 09:49:03 2026 -0700"
  },
  "message": "[compare.py] Add confidence interval (#377)\n\nThis patch adds a `--diff-confidence-interval\u003drelative|absolute` option to `compare.py` to report 95% (or 1-alpha) confidence intervals for the relative difference between `lhs` and `rhs` runs.\n\nThe current p-value and significance markers only tells the user if a difference is statistically significant against null hypothesis, but does not show how large the true difference might vary.\n\nExample output from `compare.py ... --statistics --diff-confidence-interval`\n```\n     Program                                       exec_time\n                                                   lhs       rhs    diff  std_lhs std_rhs t-value p-value significant diff_ci_rel\n     C                                               2.95      3.40 15.3% 0.076   0.100   -6.653  0.0027  Y           [ 9.3%, 22.7%]\n     A                                               1.00      1.15 15.0% 0.050   0.050   -3.674  0.0213  Y           [ 3.5%, 25.1%]\n     B                                               1.95      2.20 12.8% 0.076   0.050   -4.427  0.0114  Y           [ 4.3%, 18.8%]\n                                Geomean difference                  14.4%\n```",
  "tree_diff": [
    {
      "type": "modify",
      "old_id": "244d18faa9f6fd68b8fa78ac578b772830365845",
      "old_mode": 33261,
      "old_path": "utils/compare.py",
      "new_id": "c9f90e92d96904bab2ceeda58694b8168fc8753b",
      "new_mode": 33261,
      "new_path": "utils/compare.py"
    }
  ]
}