Update app.py

app.py

@@ -1,24 +1,223 @@
+import os
+import re
+import torch
+from collections import Counter
+from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForTokenClassification, AutoModelForCausalLM
+import PyPDF2
+import openai
+import docx
+from arabert.preprocess import ArabertPreprocessor
 import gradio as gr
-import subprocess
 
-def run_scripts():
-    outputs = []
-    scripts = ["firstkha.py", "alf.py"]
-    for script in scripts:
+
+# التحقق من توفر GPU واستخدامه
+device = 0 if torch.cuda.is_available() else -1
+
+# تحميل نماذج BERT و GPT2
+arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
+arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")
+
+arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
+arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")
+
+gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-large", trust_remote_code=True)
+gpt2_model = AutoModelForCausalLM.from_pretrained("aubmindlab/aragpt2-large", trust_remote_code=True)
+
+# إعداد المعالج النصي لـ AraBERT
+arabert_prep = ArabertPreprocessor("aubmindlab/bert-base-arabertv02")
+
+# دالة لتقسيم النص إلى أجزاء بناءً على عدد التوكنز
+def split_text_into_chunks(text, tokenizer, max_length):
+    tokens = tokenizer.tokenize(text)
+    chunks = []
+    for i in range(0, len(tokens), max_length):
+        chunk_tokens = tokens[i:i + max_length]
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
+        chunks.append(chunk_text)
+    return chunks
+
+# دالة لتجزئة النص إلى جمل باستخدام التعبيرات العادية
+def extract_sentences(text):
+    sentences = re.split(r'(?<=[.!؟]) +', text)
+    return sentences
+
+# دالة لاستخراج الاقتباسات من النص
+def extract_quotes(text):
+    quotes = re.findall(r'[“"«](.*?)[”"»]', text)
+    return quotes
+
+# دالة لعد الرموز في النص
+def count_tokens(text, tokenizer):
+    tokens = tokenizer.tokenize(text)
+    return len(tokens)
+
+# دالة لاستخراج النص من ملفات PDF
+def extract_pdf_text(file_path):
+    with open(file_path, "rb") as pdf_file:
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        text = ""
+        for page_num in range(len(pdf_reader.pages)):
+            page = pdf_reader.pages[page_num]
+            text += page.extract_text()
+    return text
+
+# دالة لاستخراج النص من ملفات DOCX
+def extract_docx_text(file_path):
+    doc = docx.Document(file_path)
+    text = "\n".join([para.text for para in doc.paragraphs])
+    return text
+
+# دالة لقراءة النص من ملف مع التعامل مع مشاكل الترميز
+def read_text_file(file_path):
+    try:
+        with open(file_path, "r", encoding="utf-8") as file:
+            return file.read()
+    except UnicodeDecodeError:
         try:
-            result = subprocess.run(["python", script], check=True, text=True, capture_output=True)
-            outputs.append(f"Output of {script}:\n{result.stdout}")
-        except subprocess.CalledProcessError as e:
-            outputs.append(f"Error running {script}:\n{e.stderr}")
-    return "\n\n".join(outputs)
-
-iface = gr.Interface(
-    fn=run_scripts,
-    inputs=[],
-    outputs="text",
-    live=True,
-    description="Run the scripts firstkha.py and alf.py"
+            with open(file_path, "r", encoding="latin-1") as file:
+                return file.read()
+        except UnicodeDecodeError:
+            with open(file_path, "r", encoding="cp1252") as file:
+                return file.read()
+
+# دالة لاستخراج المشاهد من النص
+def extract_scenes(text):
+    scenes = re.split(r'داخلي|خارجي', text)
+    scenes = [scene.strip() for scene in scenes if scene.strip()]
+    return scenes
+
+# دالة لاستخراج تفاصيل المشهد (المكان والوقت)
+def extract_scene_details(scene):
+    details = {}
+    location_match = re.search(r'(داخلي|خارجي)', scene)
+    time_match = re.search(r'(ليلاً|نهاراً|شروق|غروب)', scene)
+
+    if location_match:
+        details['location'] = location_match.group()
+    if time_match:
+        details['time'] = time_match.group()
+
+    return details
+
+# دالة لاستخراج أعمار الشخصيات
+def extract_ages(text):
+    ages = re.findall(r'\b(\d{1,2})\s*(?:عام|سنة|سنوات)\s*(?:من العمر|عمره|عمرها)', text)
+    return ages
+
+# دالة لاستخراج وصف الشخصيات
+def extract_character_descriptions(text):
+    descriptions = re.findall(r'شخصية\s*(.*?)\s*:\s*وصف\s*(.*?)\s*(?:\.|،)', text, re.DOTALL)
+    return descriptions
+
+# دالة لاستخراج تكرار الشخصيات
+def extract_character_frequency(entities):
+    persons = [ent[0] for ent in entities['PERSON']]
+    frequency = Counter(persons)
+    return frequency
+
+# دالة لاستخراج الحوارات وتحديد المتحدثين
+def extract_dialogues(text):
+    dialogues = re.findall(r'(.*?)(?:\s*:\s*)(.*?)(?=\n|$)', text, re.DOTALL)
+    return dialogues
+
+# دالة لمعالجة الملفات وتقسيمها بناءً على عدد التوكنز
+def process_files(input_directory, output_directory_950):
+    for file_name in os.listdir(input_directory):
+        file_path = os.path.join(input_directory, file_name)
+        
+        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
+            continue
+        
+        if file_path.endswith(".pdf"):
+            text = extract_pdf_text(file_path)
+        elif file_path.endswith(".docx"):
+            text = extract_docx_text(file_path)
+        else:
+            text = read_text_file(file_path)
+        
+        # تقسيم النص إلى أجزاء لا تتجاوز 950 توكنز
+        chunks_950 = split_text_into_chunks(text, gpt2_tokenizer, 950)
+        for i, chunk in enumerate(chunks_950):
+            output_file_950 = os.path.join(output_directory_950, f"{os.path.splitext(file_name)[0]}_part_{i+1}.txt")
+            with open(output_file_950, "w", encoding="utf-8") as file:
+                file.write(chunk)
+
+# دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
+def analyze_files(input_directory, output_directory, tokenizer, max_length):
+    for file_name in os.listdir(input_directory):
+        file_path = os.path.join(input_directory, file_name)
+        
+        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
+            continue
+        
+        with open(file_path, "r", encoding="utf-8") as file:
+            text = file.read()
+        
+        chunks = split_text_into_chunks(text, tokenizer, max_length)
+        
+        # إجراء التحليل على النصوص المقسمة
+        for chunk in chunks:
+            sentences = extract_sentences(chunk)
+            quotes = extract_quotes(chunk)
+            token_count = count_tokens(chunk, tokenizer)
+            scenes = extract_scenes(chunk)
+            ages = extract_ages(chunk)
+            character_descriptions = extract_character_descriptions(chunk)
+            dialogues = extract_dialogues(chunk)
+            scene_details = [extract_scene_details(scene) for scene in scenes]
+
+            # حفظ النتائج
+            with open(os.path.join(output_directory, f"{file_name}_sentences.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(sentences))
+
+          
+            with open(os.path.join(output_directory, f"{file_name}_quotes.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(quotes))
+
+            with open(os.path.join(output_directory, f"{file_name}_token_count.txt"), "a", encoding="utf-8") as file:
+                file.write(str(token_count))
+
+            with open(os.path.join(output_directory, f"{file_name}_scenes.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(scenes))
+
+            with open(os.path.join(output_directory, f"{file_name}_scene_details.txt"), "a", encoding="utf-8") as file:
+                file.write(str(scene_details))
+
+            with open(os.path.join(output_directory, f"{file_name}_ages.txt"), "a", encoding="utf-8") as file:
+                file.write(str(ages))
+
+            with open(os.path.join(output_directory, f"{file_name}_character_descriptions.txt"), "a", encoding="utf-8") as file:
+                file.write(str(character_descriptions))
+
+            with open(os.path.join(output_directory, f"{file_name}_dialogues.txt"), "a", encoding="utf-8") as file:
+                file.write(str(dialogues))
+
+# تحديد المسارات
+input_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/in"
+output_directory_950 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/1000"
+input_directory_950 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/1000"
+output_directory_950_out = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out/1000"
+
+# التأكد من وجود المسارات
+os.makedirs(output_directory_950, exist_ok=True)
+os.makedirs(output_directory_950_out, exist_ok=True)
+
+# معالجة الملفات وتقسيمها
+process_files(input_directory, output_directory_950)
+
+# تحليل الملفات المقسمة إلى 950 توكنز
+analyze_files(input_directory_950, output_directory_950_out, gpt2_tokenizer, 950)
+
+print("تمت معالجة الملفات وتحليلها بنجاح.")
+
+# تعريف واجهة Gradio
+interface = gr.Interface(
+    fn=analyze_files,
+    inputs=gr.File(file_count="multiple", type="filepath"),
+    outputs=gr.JSON(),
+    title="Movie Script Analyzer and Completer",
+    description="Upload text, PDF, or DOCX files to analyze and complete the movie script."
 )
 
 if __name__ == "__main__":
-    iface.launch()
+    interface.launch()