Upload prepare_everything.py

prepare_everything.py

@@ -0,0 +1,71 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import requests
+
+from parltopic.utils.helper import get_main_config
+
+config = get_main_config()
+
+
+def download_model(url, save_path):
+    # Send a GET request to the URL
+    response = requests.get(url, stream=True)
+
+    # Check if the request was successful (status code 200)
+    if response.status_code == 200:
+        # Open a file in binary write mode to save the downloaded content
+        with open(save_path, "wb") as f:
+            # Iterate over the response content in chunks and write to the file
+            for chunk in response.iter_content(chunk_size=1024):
+                f.write(chunk)
+        print("Model downloaded successfully!")
+    else:
+        # Print an error message if the request was not successful
+        print(f"Failed to download model. Status code: {response.status_code}")
+
+
+def set_tokenizers_parallelism(value):
+    """Set the TOKENIZERS_PARALLELISM environment variable."""
+    os.environ["TOKENIZERS_PARALLELISM"] = "true" if value else "false"
+    print(f"TOKENIZERS_PARALLELISM set to {os.environ['TOKENIZERS_PARALLELISM']}")
+
+
+def install_requirements():
+    """Install packages listed in requirements.txt"""
+    try:
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]
+        )
+        print("All packages from requirements.txt installed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install packages from requirements.txt: {e}")
+        sys.exit(1)
+
+
+def install_spacy_model(model_name):
+    """Install a specific spaCy model"""
+    try:
+        subprocess.check_call([sys.executable, "-m", "spacy", "download", model_name])
+        print(f"spaCy model '{model_name}' installed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install spaCy model '{model_name}': {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+
+    for name, path in config["paths"].items():
+        Path(path).mkdir(exist_ok=True)
+
+    download_model(
+        "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin",
+        Path(config["paths"]["resources"]) / "lid.176.bin",
+    )
+
+    install_spacy_model("de_core_news_lg")
+    install_spacy_model("fr_core_news_lg")
+    install_spacy_model("it_core_news_lg")
+    set_tokenizers_parallelism(True)