Update app.py

app.py

@@ -1,14 +1,13 @@
 import os
 import re
-import camel_tools
 from camel_tools.tokenizers.word import simple_word_tokenize
-from camel_tools.ner import NERecognizer
 import nltk
 import torch
 from collections import Counter
 from transformers import pipeline, AutoModel, AutoTokenizer
 import PyPDF2
 import gradio as gr
+import openai
 
 # تحميل وتفعيل الأدوات المطلوبة
 nltk.download('punkt')
@@ -16,29 +15,42 @@ nltk.download('punkt')
 # التحقق من توفر GPU واستخدامه
 device = 0 if torch.cuda.is_available() else -1
 
-# تحميل نماذج التحليل اللغوي
-analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english", device=device)
+# إعداد التوكنات
+openai.api_key = "sk-proj-62TDbO5KABSdkZaFPPD4T3BlbkFJkhqOYpHhL6OucTzNdWSU"
 
-# تحميل نموذج التعرف على الكيانات في camel_tools
-ner = NERecognizer.pretrained()
+# تحميل نماذج التحليل اللغوي
+analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english", device=device, use_auth_token=huggingface_token)
 
 # تحميل نماذج BERT، GPT2، ELECTRA، و AraBERT
-arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
-arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")
+arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic", use_auth_token=huggingface_token)
+arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic", use_auth_token=huggingface_token)
+
+arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base", use_auth_token=huggingface_token)
+arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base", use_auth_token=huggingface_token)
+
+arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator", use_auth_token=huggingface_token)
+arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator", use_auth_token=huggingface_token)
+
+arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02", use_auth_token=huggingface_token)
+arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02", use_auth_token=huggingface_token)
 
-arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base")
-arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base")
+aragpt2_mega_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-mega", use_auth_token=huggingface_token)
+aragpt2_mega_model = AutoModel.from_pretrained("aubmindlab/aragpt2-mega", use_auth_token=huggingface_token)
 
-arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator")
-arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator")
+xlm_roberta_tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large", use_auth_token=huggingface_token)
+xlm_roberta_model = AutoModel.from_pretrained("xlm-roberta-large", use_auth_token=huggingface_token)
 
-arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")
+m2m100_tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M", use_auth_token=huggingface_token)
+m2m100_model = AutoModel.from_pretrained("facebook/m2m100_418M", use_auth_token=huggingface_token)
 
-# دالة لتحليل النص باستخدام camel_tools
+# دالة لتحليل النص باستخدام arabert-ner من transformers
 def camel_ner_analysis(text):
+    tokenizer = AutoTokenizer.from_pretrained("camel-ai/arabert-ner", use_auth_token=huggingface_token)
+    model = AutoModel.from_pretrained("camel-ai/arabert-ner", use_auth_token=huggingface_token)
     tokens = simple_word_tokenize(text)
-    entities = ner.predict(tokens)
+    inputs = tokenizer(tokens, return_tensors="pt", is_split_into_words=True)
+    outputs = model(**inputs)
+    entities = outputs.logits.argmax(dim=-1).squeeze().tolist()
     entity_dict = {"PERSON": [], "LOC": [], "ORG": [], "DATE": []}
     for token, tag in zip(tokens, entities):
         if tag in entity_dict:
@@ -122,7 +134,7 @@ def extract_dialogues(text):
 # دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
 def analyze_and_complete(file_paths):
     results = []
-    output_directory = os.getenv("SPACE_DIR", "/Volumes/CLOCKWORK T/clockworkspace/first pro")
+    output_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out"
     
     for file_path in file_paths:
         if file_path.endswith(".pdf"):