Steven Tang
commited on
Commit
·
57ff211
1
Parent(s):
4da4ff3
Fixes to run_batches
Browse files- run_batches.py +6 -4
run_batches.py
CHANGED
@@ -39,6 +39,8 @@ def main():
|
|
39 |
input_files_batch = list(input_files)[: args.batch_size]
|
40 |
|
41 |
with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
|
|
|
|
|
42 |
for file in input_files_batch:
|
43 |
shutil.copy(input_dir / file, input_temp_dir)
|
44 |
|
@@ -60,11 +62,11 @@ def main():
|
|
60 |
logging.exception(f"Error running command: {command}")
|
61 |
raise e
|
62 |
|
63 |
-
|
64 |
output_files = set(file.name for file in output_files)
|
65 |
-
for
|
66 |
-
shutil.copy(
|
67 |
-
input_files.remove(
|
68 |
end = time.time()
|
69 |
logging.info(f"Processed {len(output_files)} files in {timedelta(seconds=end - start)}")
|
70 |
|
|
|
39 |
input_files_batch = list(input_files)[: args.batch_size]
|
40 |
|
41 |
with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
|
42 |
+
input_temp_dir = Path(input_temp_dir)
|
43 |
+
output_temp_dir = Path(output_temp_dir)
|
44 |
for file in input_files_batch:
|
45 |
shutil.copy(input_dir / file, input_temp_dir)
|
46 |
|
|
|
62 |
logging.exception(f"Error running command: {command}")
|
63 |
raise e
|
64 |
|
65 |
+
output_paths = output_temp_dir.rglob("*")
|
66 |
output_files = set(file.name for file in output_files)
|
67 |
+
for output_path, output_file in zip(output_paths, output_files):
|
68 |
+
shutil.copy(output_path, output_dir)
|
69 |
+
input_files.remove(output_file)
|
70 |
end = time.time()
|
71 |
logging.info(f"Processed {len(output_files)} files in {timedelta(seconds=end - start)}")
|
72 |
|