tools/flakiness/find_flakiness.py
def FindUnaryFlakiness(test_path, output_path, num_procs, num_repeats, timeout):
  """Runs all the test cases in a given test in parallel with itself, to get at
  those that hold on to shared resources. The idea is that if a test uses a
  unary resource, then running many instances of this test will purge out some
  of them as failures or timeouts.
  """

  test_name_regex = r'((\w+/)?\w+\.\w+(/\d+)?)'
  test_start = re.compile('\[\s+RUN\s+\] ' + test_name_regex)
  test_list = []

  # Run the test to discover all the test cases.
  proc = subprocess.Popen([test_path], stdout=subprocess.PIPE)
  while True:
    line = proc.stdout.readline()
    if not line:
      if proc.poll() is not None:
        break
      continue
    print(line.rstrip())
    results = test_start.search(line)
    if results:
      test_list.append(results.group(1))

  failures = []
  index = 0
  total = len(test_list)

  # Run each test case in parallel with itself.
  for test_name in test_list:
    num_fails = 0
    num_terminated = 0
    procs = []
    args = [test_path, '--gtest_filter=' + test_name,
            '--gtest_repeat=%i' % num_repeats]
    while len(procs) < num_procs:
      procs.append(subprocess.Popen(args))
    seconds = 0
    while procs:
      for proc in procs:
        if proc.poll() is not None:
          if proc.returncode != 0:
            ++num_fails
          procs.remove(proc)
      # Timeout exceeded, kill the remaining processes and make a note.
      if seconds > timeout:
        num_fails += len(procs)
        num_terminated = len(procs)
        while procs:
          procs.pop().terminate()
      time.sleep(1.0)
      seconds += 1
    if num_fails:
      line = '%s: %i failed' % (test_name, num_fails)
      if num_terminated:
        line += ' (%i terminated)' % num_terminated
      failures.append(line)
    print('%s (%i / %i): %i failed' % (test_name, index, total, num_fails))
    index += 1
    time.sleep(1.0)

  # Print the results and write the data file.
  print(failures)
  data_file = open(output_path, 'w')
  for line in failures:
    data_file.write(line + '\n')
  data_file.close()
Runs all the test cases in a given test in parallel with itself, to get at
  those that hold on to shared resources. The idea is that if a test uses a
  unary resource, then running many instances of this test will purge out some
  of them as failures or timeouts.
  
Runs all the test cases in a given test in parallel with itself, to get at
those that hold on to shared resources. The idea is that if a test uses a
unary resource, then running many instances of this test will purge out some
of them as failures or timeouts.
Runs all the test cases in a given test in parallel with itself to get at those that hold on to shared resources . The idea is that if a test uses a unary resource then running many instances of this test will purge out some of them as failures or timeouts .
def FindUnaryFlakiness(test_path, output_path, num_procs, num_repeats, timeout):
  test_name_regex = r'((\w+/)?\w+\.\w+(/\d+)?)'
  test_start = re.compile('\[\s+RUN\s+\] ' + test_name_regex)
  test_list = []
  proc = subprocess.Popen([test_path], stdout=subprocess.PIPE)
  while True:
    line = proc.stdout.readline()
    if not line:
      if proc.poll() is not None:
        break
      continue
    print(line.rstrip())
    results = test_start.search(line)
    if results:
      test_list.append(results.group(1))
  failures = []
  index = 0
  total = len(test_list)
  for test_name in test_list:
    num_fails = 0
    num_terminated = 0
    procs = []
    args = [test_path, '--gtest_filter=' + test_name,
            '--gtest_repeat=%i' % num_repeats]
    while len(procs) < num_procs:
      procs.append(subprocess.Popen(args))
    seconds = 0
    while procs:
      for proc in procs:
        if proc.poll() is not None:
          if proc.returncode != 0:
            ++num_fails
          procs.remove(proc)
      if seconds > timeout:
        num_fails += len(procs)
        num_terminated = len(procs)
        while procs:
          procs.pop().terminate()
      time.sleep(1.0)
      seconds += 1
    if num_fails:
      line = '%s: %i failed' % (test_name, num_fails)
      if num_terminated:
        line += ' (%i terminated)' % num_terminated
      failures.append(line)
    print('%s (%i / %i): %i failed' % (test_name, index, total, num_fails))
    index += 1
    time.sleep(1.0)
  print(failures)
  data_file = open(output_path, 'w')
  for line in failures:
    data_file.write(line + '\n')
  data_file.close()
def FindUnaryFlakiness ( test_path , output_path , num_procs , num_repeats , timeout ) : test_name_regex = r'((\w+/)?\w+\.\w+(/\d+)?)' test_start = re . compile ( '\[\s+RUN\s+\] ' + test_name_regex ) test_list = [ ] proc = subprocess . Popen ( [ test_path ] , stdout = subprocess . PIPE ) while True : line = proc . stdout . readline ( ) if not line : if proc . poll ( ) is not None : break continue print ( line . rstrip ( ) ) results = test_start . search ( line ) if results : test_list . append ( results . group ( 1 ) ) failures = [ ] index = 0 total = len ( test_list ) for test_name in test_list : num_fails = 0 num_terminated = 0 procs = [ ] args = [ test_path , '--gtest_filter=' + test_name , '--gtest_repeat=%i' % num_repeats ] while len ( procs ) < num_procs : procs . append ( subprocess . Popen ( args ) ) seconds = 0 while procs : for proc in procs : if proc . poll ( ) is not None : if proc . returncode != 0 : + + num_fails procs . remove ( proc ) if seconds > timeout : num_fails += len ( procs ) num_terminated = len ( procs ) while procs : procs . pop ( ) . terminate ( ) time . sleep ( 1.0 ) seconds += 1 if num_fails : line = '%s: %i failed' % ( test_name , num_fails ) if num_terminated : line += ' (%i terminated)' % num_terminated failures . append ( line ) print ( '%s (%i / %i): %i failed' % ( test_name , index , total , num_fails ) ) index += 1 time . sleep ( 1.0 ) print ( failures ) data_file = open ( output_path , 'w' ) for line in failures : data_file . write ( line + '\n' ) data_file . close ( )
Runs all the test cases in a given test in parallel with itself, to get at
those that hold on to shared resources.
Runs all the test cases in a given test in parallel with itself to get at those that hold on to shared resources .
"""Runs all the test cases in a given test in parallel with itself, to get at
  those that hold on to shared resources. The idea is that if a test uses a
  unary resource, then running many instances of this test will purge out some
  of them as failures or timeouts.
  """ # Run the test to discover all the test cases. # Run each test case in parallel with itself. # Timeout exceeded, kill the remaining processes and make a note. # Print the results and write the data file.
test_path
output_path
num_procs
num_repeats
timeout