Upload folder using huggingface_hub

run_batches.py

@@ -84,6 +84,9 @@ def run_batch(input_files_batch, input_dir, output_dir, mode):
                 subprocess.run([command], check=True, shell=True)
             except subprocess.CalledProcessError as e:
                 logging.exception(f"Error running command: {command}")
+                if "returned non-zero exit status 137" in str(e):
+                    logging.error("Process killed due to memory limit")
+                    return
                 raise e
 
         output_paths = list(output_temp_dir.rglob("*"))