labtools/_src/util.py
def catch_exp_failures(name: str, verbose: bool = True):
  """ Context manager for catching experiment failures.

    This context manager will ignore all exceptions within context except
    Keyboard Interrupts and is useful for running sets of experiments. Failed
    experiments will be logged to stderr (absl) and skipped. If torch is
    installed, we empty the catch on exit of the context manager. We also run
    Garbage collection. This should cleanup most GPU allocations, and is known
    to work with Jax, PyTorch, and DALI.

    Examples:
      >>> with catch_exp_failures('Model'):
      ...   lambda : raise RuntimeError
      Failed to run Model after * s. Skipping...
      >>> with catch_exp_failures('Model'):
      ...   lambda : None
      Succesfully ran Model in * s.

    Args:
      name: Name of the experiment
      verbose: Predicate indicating whether to log details about timing
        information of the experiment on success.
  """
  tick = time.time()
  try:
    yield
    if verbose:
      logging.info('Succesfully ran %s in %0.3d s.', name, time.time() - tick)
  # fail on KeyboardInterrupt
  except KeyboardInterrupt:
    logging.info('Detected KeyboardInterrupt, exiting.')
    import os
    os._exit(1)
  # catch other errors
  except Exception:
    logging.exception('Failed to run %s after %s s. Skipping...', name,
                      time.time() - tick)
  finally:
    # maybe emty torch cuda cache
    if is_installed('torch'):
      import torch
      torch.cuda.empty_cache()
    gc.collect()
 Context manager for catching experiment failures.

    This context manager will ignore all exceptions within context except
    Keyboard Interrupts and is useful for running sets of experiments. Failed
    experiments will be logged to stderr (absl) and skipped. If torch is
    installed, we empty the catch on exit of the context manager. We also run
    Garbage collection. This should cleanup most GPU allocations, and is known
    to work with Jax, PyTorch, and DALI.

    Examples:
      >>> with catch_exp_failures('Model'):
      ...   lambda : raise RuntimeError
      Failed to run Model after * s. Skipping...
      >>> with catch_exp_failures('Model'):
      ...   lambda : None
      Succesfully ran Model in * s.

    Args:
      name: Name of the experiment
      verbose: Predicate indicating whether to log details about timing
        information of the experiment on success.
  
Context manager for catching experiment failures.
This context manager will ignore all exceptions within context except
Keyboard Interrupts and is useful for running sets of experiments. Failed
experiments will be logged to stderr (absl) and skipped. If torch is
installed, we empty the catch on exit of the context manager. We also run
Garbage collection. This should cleanup most GPU allocations, and is known
to work with Jax, PyTorch, and DALI.
Context manager for catching experiment failures . This context manager will ignore all exceptions within context except Keyboard Interrupts and is useful for running sets of experiments . Failed experiments will be logged to stderr ( absl ) and skipped . If torch is installed we empty the catch on exit of the context manager . We also run Garbage collection . This should cleanup most GPU allocations and is known to work with Jax PyTorch and DALI .
def catch_exp_failures(name: str, verbose: bool = True):
  tick = time.time()
  try:
    yield
    if verbose:
      logging.info('Succesfully ran %s in %0.3d s.', name, time.time() - tick)
  except KeyboardInterrupt:
    logging.info('Detected KeyboardInterrupt, exiting.')
    import os
    os._exit(1)
  except Exception:
    logging.exception('Failed to run %s after %s s. Skipping...', name,
                      time.time() - tick)
  finally:
    if is_installed('torch'):
      import torch
      torch.cuda.empty_cache()
    gc.collect()
def catch_exp_failures ( name : str , verbose : bool = True ) : tick = time . time ( ) try : yield if verbose : logging . info ( 'Succesfully ran %s in %0.3d s.' , name , time . time ( ) - tick ) except KeyboardInterrupt : logging . info ( 'Detected KeyboardInterrupt, exiting.' ) import os os . _exit ( 1 ) except Exception : logging . exception ( 'Failed to run %s after %s s. Skipping...' , name , time . time ( ) - tick ) finally : if is_installed ( 'torch' ) : import torch torch . cuda . empty_cache ( ) gc . collect ( )
Context manager for catching experiment failures.
Context manager for catching experiment failures .
""" Context manager for catching experiment failures.

    This context manager will ignore all exceptions within context except
    Keyboard Interrupts and is useful for running sets of experiments. Failed
    experiments will be logged to stderr (absl) and skipped. If torch is
    installed, we empty the catch on exit of the context manager. We also run
    Garbage collection. This should cleanup most GPU allocations, and is known
    to work with Jax, PyTorch, and DALI.

    Examples:
      >>> with catch_exp_failures('Model'):
      ...   lambda : raise RuntimeError
      Failed to run Model after * s. Skipping...
      >>> with catch_exp_failures('Model'):
      ...   lambda : None
      Succesfully ran Model in * s.

    Args:
      name: Name of the experiment
      verbose: Predicate indicating whether to log details about timing
        information of the experiment on success.
  """ # fail on KeyboardInterrupt # catch other errors # maybe emty torch cuda cache
name
str
Name of the experiment
Name of the experiment
verbose
bool
Predicate indicating whether to log details about timing
information of the experiment on success.
Predicate indicating whether to log details about timing information of the experiment on success .
examples
>>> with catch_exp_failures('Model'):
lambda : raise RuntimeError
Failed to run Model after * s. Skipping
>>> with catch_exp_failures('Model'):
lambda : None
Succesfully ran Model in * s.
>>> with catch_exp_failures ( ' Model ' ) : lambda : raise RuntimeError Failed to run Model after * s . Skipping >>> with catch_exp_failures ( ' Model ' ) : lambda : None Succesfully ran Model in * s .