utils/release/findRegressions-simple.py

#!/usr/bin/env python

from __future__ import print_function
import re, string, sys, os, time, math

DEBUG = 0

(tp, exp) = ('compile', 'exec')

def parse(file):
  f = open(file, 'r')
  d = f.read()
  
  # Cleanup weird stuff
  d = re.sub(r',\d+:\d', '', d)

  r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)

  test = {}
  fname = ''
  for t in r:
    if DEBUG:
      print(t)

    if t[0] == 'PASS' or t[0] == 'FAIL' :
      tmp = t[2].split('llvm-test/')
      
      if DEBUG:
        print(tmp)

      if len(tmp) == 2:
        fname = tmp[1].strip('\r\n')
      else:
        fname = tmp[0].strip('\r\n')

      if fname not in test:
        test[fname] = {}

      test[fname][t[1] + ' state'] = t[0]
      test[fname][t[1] + ' time'] = float('nan')
    else :
      try:
        n = t[0].split('RESULT-')[1]

        if DEBUG:
          print("n == ", n);
        
        if n == 'compile-success':
          test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))

        elif n == 'exec-success':
          test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
          if DEBUG:
            print(test[fname][string.replace(n, '-success', '')])

        else :
          # print "ERROR!"
          sys.exit(1)

      except:
          continue

  return test

# Diff results and look for regressions.
def diffResults(d_old, d_new):
  regressions = {}
  passes = {}
  removed = ''

  for x in ['compile state', 'compile time', 'exec state', 'exec time']:
    regressions[x] = ''
    passes[x] = ''

  for t in sorted(d_old.keys()) :
    if t in d_new:

      # Check if the test passed or failed.
      for x in ['compile state', 'compile time', 'exec state', 'exec time']:

        if x not in d_old[t] and x not in d_new[t]:
          continue

        if x in d_old[t]:
          if x in d_new[t]:

            if d_old[t][x] == 'PASS':
              if d_new[t][x] != 'PASS':
                regressions[x] += t + "\n"
            else:
              if d_new[t][x] == 'PASS':
                passes[x] += t + "\n"

          else :
            regressions[x] += t + "\n"

        if x == 'compile state' or x == 'exec state':
          continue

        # For execution time, if there is no result it's a fail.
        if x not in d_old[t] and x not in d_new[t]:
          continue
        elif x not in d_new[t]:
          regressions[x] += t + "\n"
        elif x not in d_old[t]:
          passes[x] += t + "\n"

        if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
          continue

        elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
          passes[x] += t + "\n"

        elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
          regressions[x] += t + ": NaN%\n"

        if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \
              (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05:
          regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n"

    else :
      removed += t + "\n"

  if len(regressions['compile state']) != 0:
    print('REGRESSION: Compilation Failed')
    print(regressions['compile state'])

  if len(regressions['exec state']) != 0:
    print('REGRESSION: Execution Failed')
    print(regressions['exec state'])

  if len(regressions['compile time']) != 0:
    print('REGRESSION: Compilation Time')
    print(regressions['compile time'])

  if len(regressions['exec time']) != 0:
    print('REGRESSION: Execution Time')
    print(regressions['exec time'])

  if len(passes['compile state']) != 0:
    print('NEW PASSES: Compilation')
    print(passes['compile state'])

  if len(passes['exec state']) != 0:
    print('NEW PASSES: Execution')
    print(passes['exec state'])

  if len(removed) != 0:
    print('REMOVED TESTS')
    print(removed)

# Main
if len(sys.argv) < 3 :
  print('Usage:', sys.argv[0], '<old log> <new log>')
  sys.exit(-1)

d_old = parse(sys.argv[1])
d_new = parse(sys.argv[2])

diffResults(d_old, d_new)