hf http cache cleanup

app.py

@@ -15,7 +15,7 @@ from interfaces.emotion9 import demo as e9_demo
 from interfaces.cap_media_demo import demo as cap_media_demo
 from interfaces.cap_media2 import demo as cap_media2_demo
 from interfaces.cap_minor_media import demo as cap_minor_media_demo
-from utils import download_hf_models, df_h, set_hf_cache_dir, scan_cache
+from utils import download_hf_models, hf_cleanup, df_h, set_hf_cache_dir, scan_cache
 
 
 
@@ -54,10 +54,12 @@ with gr.Blocks(css=css) as demo:
 
 if __name__ == "__main__":
     set_hf_cache_dir("/data")
+    hf_cleanup()
     df_h() # debug -> check disk space before launching demo - TO-DO: smarter disk space usage
     scan_cache()
     download_spacy_models() 
     download_hf_models() # does this affect the build?
+    hf_cleanup()
     demo.launch()
 
 # TODO: add all languages & domains

utils.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import glob
 import subprocess
 
 import torch
@@ -114,6 +115,15 @@ def df_h():
     print("=== Disk Usage for /data/ (du -h --max-depth=2) ===")
     print(du_result.stdout)
     
+    
+def hf_cleanup():
+    http_folders = glob.glob("/data/http*")
+    for folder in http_folders:
+        if os.path.isdir(folder):
+            print(f"Deleting: {folder}")
+            shutil.rmtree(folder)
+    
+    
 def scan_cache():
     # Scan Hugging Face model cache
     cache_dir = os.environ.get("TRANSFORMERS_CACHE", os.path.expanduser("~/.cache/huggingface/transformers"))