js/src/tests/compare_bench.py

   1 #!/usr/bin/env python2.4
   2 """usage: %progname candidate_path baseline_path
   3 """
   4
   5 import json
   6 import optparse
   7 from operator import itemgetter
   8
   9
  10 def avg(seq):
  11     return sum(seq) / len(seq)
  12
  13
  14 def compare(current, baseline):
  15     percent_speedups = []
  16     for key, current_result in iter(current.items()):
  17         try:
  18             baseline_result = baseline[key]
  19         except KeyError:
  20             print(key, "missing from baseline")
  21             continue
  22         val_getter = itemgetter("average_ms", "stddev_ms")
  23         base_avg, base_stddev = val_getter(baseline_result)
  24         current_avg, current_stddev = val_getter(current_result)
  25         t_best = current_avg - current_stddev
  26         t_worst = current_avg + current_stddev
  27         base_t_best = base_avg - base_stddev
  28         base_t_worst = base_avg + base_stddev
  29         if t_worst < base_t_best:
  30             # Worst takes less time (better) than baseline's best.
  31             speedup = -((t_worst - base_t_best) / base_t_best) * 100
  32             result = "faster: {:6.2f}ms < baseline {:6.2f}ms ({:+6.2f}%)".format(
  33                 t_worst, base_t_best, speedup
  34             )
  35             percent_speedups.append(speedup)
  36         elif t_best > base_t_worst:
  37             # Best takes more time (worse) than baseline's worst.
  38             slowdown = -((t_best - base_t_worst) / base_t_worst) * 100
  39             result = "SLOWER: {:6.2f}ms > baseline {:6.2f}ms ({:+6.2f}%) ".format(
  40                 t_best, base_t_worst, slowdown
  41             )
  42             percent_speedups.append(slowdown)
  43         else:
  44             result = "Meh."
  45         print("{:30s}: {}".format(key, result))
  46     if percent_speedups:
  47         print("Average speedup: {:.2f}%".format(avg(percent_speedups)))
  48
  49
  50 def compare_immediate(current_map, baseline_path):
  51     baseline_file = open(baseline_path)
  52     baseline_map = json.load(baseline_file)
  53     baseline_file.close()
  54     compare(current_map, baseline_map)
  55
  56
  57 def main(candidate_path, baseline_path):
  58     candidate_file, baseline_file = open(candidate_path), open(baseline_path)
  59     candidate = json.load(candidate_file)
  60     baseline = json.load(baseline_file)
  61     compare(candidate, baseline)
  62     candidate_file.close()
  63     baseline_file.close()
  64
  65
  66 if __name__ == "__main__":
  67     parser = optparse.OptionParser(usage=__doc__.strip())
  68     options, args = parser.parse_args()
  69     try:
  70         candidate_path = args.pop(0)
  71     except IndexError:
  72         parser.error("A JSON filepath to compare against baseline is required")
  73     try:
  74         baseline_path = args.pop(0)
  75     except IndexError:
  76         parser.error("A JSON filepath for baseline is required")
  77     main(candidate_path, baseline_path)