Steven Tang commited on
Commit
57ff211
·
1 Parent(s): 4da4ff3

Fixes to run_batches

Browse files
Files changed (1) hide show
  1. run_batches.py +6 -4
run_batches.py CHANGED
@@ -39,6 +39,8 @@ def main():
39
  input_files_batch = list(input_files)[: args.batch_size]
40
 
41
  with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
 
 
42
  for file in input_files_batch:
43
  shutil.copy(input_dir / file, input_temp_dir)
44
 
@@ -60,11 +62,11 @@ def main():
60
  logging.exception(f"Error running command: {command}")
61
  raise e
62
 
63
- output_files = output_temp_dir.rglob("*")
64
  output_files = set(file.name for file in output_files)
65
- for file in output_files:
66
- shutil.copy(file, output_dir)
67
- input_files.remove(file)
68
  end = time.time()
69
  logging.info(f"Processed {len(output_files)} files in {timedelta(seconds=end - start)}")
70
 
 
39
  input_files_batch = list(input_files)[: args.batch_size]
40
 
41
  with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
42
+ input_temp_dir = Path(input_temp_dir)
43
+ output_temp_dir = Path(output_temp_dir)
44
  for file in input_files_batch:
45
  shutil.copy(input_dir / file, input_temp_dir)
46
 
 
62
  logging.exception(f"Error running command: {command}")
63
  raise e
64
 
65
+ output_paths = output_temp_dir.rglob("*")
66
  output_files = set(file.name for file in output_files)
67
+ for output_path, output_file in zip(output_paths, output_files):
68
+ shutil.copy(output_path, output_dir)
69
+ input_files.remove(output_file)
70
  end = time.time()
71
  logging.info(f"Processed {len(output_files)} files in {timedelta(seconds=end - start)}")
72