tools/flakiness/is_flaky.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs a test repeatedly to measure its flakiness. The return code is non-zero
   7 if the failure rate is higher than the specified threshold, but is not 100%."""
   8
   9 import argparse
  10 import multiprocessing.dummy
  11 import subprocess
  12 import sys
  13 import time
  14
  15 def load_options():
  16   parser = argparse.ArgumentParser(description=__doc__)
  17   parser.add_argument('--retries', default=1000, type=int,
  18                       help='Number of test retries to measure flakiness.')
  19   parser.add_argument('--threshold', default=0.05, type=float,
  20                       help='Minimum flakiness level at which test is '
  21                            'considered flaky.')
  22   parser.add_argument('--jobs', '-j', type=int, default=1,
  23                       help='Number of parallel jobs to run tests.')
  24   parser.add_argument('command', nargs='+', help='Command to run test.')
  25   return parser.parse_args()
  26
  27 def run_test(job):
  28   print 'Starting retry attempt %d out of %d' % (job['index'] + 1,
  29                                                  job['retries'])
  30   return subprocess.check_call(job['cmd'], stdout=subprocess.PIPE,
  31                                stderr=subprocess.STDOUT)
  32
  33 def main():
  34   options = load_options()
  35   num_passed = num_failed = 0
  36   running = []
  37
  38   pool = multiprocessing.dummy.Pool(processes=options.jobs)
  39   args = [{'index': index, 'retries': options.retries, 'cmd': options.command}
  40           for index in range(options.retries)]
  41   results = pool.map(run_test, args)
  42   num_passed = len([retcode for retcode in results if retcode == 0])
  43   num_failed = len(results) - num_passed
  44
  45   if num_passed == 0:
  46     flakiness = 0
  47   else:
  48     flakiness = num_failed / float(len(results))
  49
  50   print 'Flakiness is %.2f' % flakiness
  51   if flakiness > options.threshold:
  52     return 1
  53   else:
  54     return 0
  55
  56
  57 if __name__ == '__main__':
  58   sys.exit(main())