Upload 2 files

alf.py

@@ -0,0 +1,212 @@
+import os
+import re
+import torch
+from collections import Counter
+from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForCausalLM
+import PyPDF2
+import openai
+import docx
+from arabert.preprocess import ArabertPreprocessor
+
+
+
+
+# التحقق من توفر GPU واستخدامه
+device = 0 if torch.cuda.is_available() else -1
+
+# تحميل نماذج BERT و GPT2
+arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
+arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")
+
+arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
+arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")
+
+gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-large", trust_remote_code=True)
+gpt2_model = AutoModelForCausalLM.from_pretrained("aubmindlab/aragpt2-large", trust_remote_code=True)
+
+# إعداد المعالج النصي لـ AraBERT
+arabert_prep = ArabertPreprocessor("aubmindlab/bert-base-arabertv02")
+
+# دالة لتقسيم النص إلى أجزاء بناءً على عدد التوكنز
+def split_text_into_chunks(text, tokenizer, max_length):
+    tokens = tokenizer.tokenize(text)
+    chunks = []
+    for i in range(0, len(tokens), max_length):
+        chunk_tokens = tokens[i:i + max_length]
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
+        chunks.append(chunk_text)
+    return chunks
+
+# دالة لتجزئة النص إلى جمل باستخدام التعبيرات العادية
+def extract_sentences(text):
+    sentences = re.split(r'(?<=[.!؟]) +', text)
+    return sentences
+
+# دالة لاستخراج الاقتباسات من النص
+def extract_quotes(text):
+    quotes = re.findall(r'[“"«](.*?)[”"»]', text)
+    return quotes
+
+# دالة لعد الرموز في النص
+def count_tokens(text, tokenizer):
+    tokens = tokenizer.tokenize(text)
+    return len(tokens)
+
+# دالة لاستخراج النص من ملفات PDF
+def extract_pdf_text(file_path):
+    with open(file_path, "rb") as pdf_file:
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        text = ""
+        for page_num in range(len(pdf_reader.pages)):
+            page = pdf_reader.pages[page_num]
+            text += page.extract_text()
+    return text
+
+# دالة لاستخراج النص من ملفات DOCX
+def extract_docx_text(file_path):
+    doc = docx.Document(file_path)
+    text = "\n".join([para.text for para in doc.paragraphs])
+    return text
+
+# دالة لقراءة النص من ملف مع التعامل مع مشاكل الترميز
+def read_text_file(file_path):
+    try:
+        with open(file_path, "r", encoding="utf-8") as file:
+            return file.read()
+    except UnicodeDecodeError:
+        try:
+            with open(file_path, "r", encoding="latin-1") as file:
+                return file.read()
+        except UnicodeDecodeError:
+            with open(file_path, "r", encoding="cp1252") as file:
+                return file.read()
+
+# دالة لاستخراج المشاهد من النص
+def extract_scenes(text):
+    scenes = re.split(r'داخلي|خارجي', text)
+    scenes = [scene.strip() for scene in scenes if scene.strip()]
+    return scenes
+
+# دالة لاستخراج تفاصيل المشهد (المكان والوقت)
+def extract_scene_details(scene):
+    details = {}
+    location_match = re.search(r'(داخلي|خارجي)', scene)
+    time_match = re.search(r'(ليلاً|نهاراً|شروق|غروب)', scene)
+
+    if location_match:
+        details['location'] = location_match.group()
+    if time_match:
+        details['time'] = time_match.group()
+
+    return details
+
+# دالة لاستخراج أعمار الشخصيات
+def extract_ages(text):
+    ages = re.findall(r'\b(\d{1,2})\s*(?:عام|سنة|سنوات)\s*(?:من العمر|عمره|عمرها)', text)
+    return ages
+
+# دالة لاستخراج وصف الشخصيات
+def extract_character_descriptions(text):
+    descriptions = re.findall(r'شخصية\s*(.*?)\s*:\s*وصف\s*(.*?)\s*(?:\.|،)', text, re.DOTALL)
+    return descriptions
+
+# دالة لاستخراج تكرار الشخصيات
+def extract_character_frequency(entities):
+    persons = [ent[0] for ent in entities['PERSON']]
+    frequency = Counter(persons)
+    return frequency
+
+# دالة لاستخراج الحوارات وتحديد المتحدثين
+def extract_dialogues(text):
+    dialogues = re.findall(r'(.*?)(?:\s*:\s*)(.*?)(?=\n|$)', text, re.DOTALL)
+    return dialogues
+
+# دالة لمعالجة الملفات وتقسيمها بناءً على عدد التوكنز
+def process_files(input_directory, output_directory_950):
+    for file_name in os.listdir(input_directory):
+        file_path = os.path.join(input_directory, file_name)
+        
+        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
+            continue
+        
+        if file_path.endswith(".pdf"):
+            text = extract_pdf_text(file_path)
+        elif file_path.endswith(".docx"):
+            text = extract_docx_text(file_path)
+        else:
+            text = read_text_file(file_path)
+        
+        # تقسيم النص إلى أجزاء لا تتجاوز 950 توكنز
+        chunks_950 = split_text_into_chunks(text, gpt2_tokenizer, 950)
+        for i, chunk in enumerate(chunks_950):
+            output_file_950 = os.path.join(output_directory_950, f"{os.path.splitext(file_name)[0]}_part_{i+1}.txt")
+            with open(output_file_950, "w", encoding="utf-8") as file:
+                file.write(chunk)
+
+# دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
+def analyze_files(input_directory, output_directory, tokenizer, max_length):
+    for file_name in os.listdir(input_directory):
+        file_path = os.path.join(input_directory, file_name)
+        
+        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
+            continue
+        
+        with open(file_path, "r", encoding="utf-8") as file:
+            text = file.read()
+        
+        chunks = split_text_into_chunks(text, tokenizer, max_length)
+        
+        # إجراء التحليل على النصوص المقسمة
+        for chunk in chunks:
+            sentences = extract_sentences(chunk)
+            quotes = extract_quotes(chunk)
+            token_count = count_tokens(chunk, tokenizer)
+            scenes = extract_scenes(chunk)
+            ages = extract_ages(chunk)
+            character_descriptions = extract_character_descriptions(chunk)
+            dialogues = extract_dialogues(chunk)
+            scene_details = [extract_scene_details(scene) for scene in scenes]
+
+            # حفظ النتائج
+            with open(os.path.join(output_directory, f"{file_name}_sentences.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(sentences))
+
+          
+            with open(os.path.join(output_directory, f"{file_name}_quotes.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(quotes))
+
+            with open(os.path.join(output_directory, f"{file_name}_token_count.txt"), "a", encoding="utf-8") as file:
+                file.write(str(token_count))
+
+            with open(os.path.join(output_directory, f"{file_name}_scenes.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(scenes))
+
+            with open(os.path.join(output_directory, f"{file_name}_scene_details.txt"), "a", encoding="utf-8") as file:
+                file.write(str(scene_details))
+
+            with open(os.path.join(output_directory, f"{file_name}_ages.txt"), "a", encoding="utf-8") as file:
+                file.write(str(ages))
+
+            with open(os.path.join(output_directory, f"{file_name}_character_descriptions.txt"), "a", encoding="utf-8") as file:
+                file.write(str(character_descriptions))
+
+            with open(os.path.join(output_directory, f"{file_name}_dialogues.txt"), "a", encoding="utf-8") as file:
+                file.write(str(dialogues))
+
+# تحديد المسارات
+input_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/in"
+output_directory_950 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/1000"
+input_directory_950 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/1000"
+output_directory_950_out = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out/1000"
+
+# التأكد من وجود المسارات
+os.makedirs(output_directory_950, exist_ok=True)
+os.makedirs(output_directory_950_out, exist_ok=True)
+
+# معالجة الملفات وتقسيمها
+process_files(input_directory, output_directory_950)
+
+# تحليل الملفات المقسمة إلى 950 توكنز
+analyze_files(input_directory_950, output_directory_950_out, gpt2_tokenizer, 950)
+
+print("تمت معالجة الملفات وتحليلها بنجاح.")

firstkha.py

@@ -0,0 +1,231 @@
+import os
+import re
+import torch
+from collections import Counter
+from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForTokenClassification
+import PyPDF2
+import openai
+import docx
+
+
+# التحقق من توفر GPU واستخدامه
+device = 0 if torch.cuda.is_available() else -1
+
+# تحميل نماذج BERT، GPT2، ELECTRA، و AraBERT
+arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
+arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")
+
+arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base")
+arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base")
+
+arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator")
+arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator")
+
+arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
+arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")
+
+# تحميل نموذج التعرف على الكيانات المسماة من CAMeL-Lab
+ner_tokenizer = AutoTokenizer.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
+ner_model = AutoModelForTokenClassification.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
+nlp_ner = pipeline("ner", model=ner_model, tokenizer=ner_tokenizer)
+
+# دالة لتحليل النص باستخدام transformers
+def camel_ner_analysis(text):
+    ner_results = nlp_ner(text)
+    entity_dict = {"PERSON": [], "LOC": [], "ORG": [], "DATE": []}
+    for entity in ner_results:
+        entity_type = entity["entity"]
+        if entity_type in entity_dict:
+            entity_dict[entity_type].append((entity["word"], entity_type))
+    return entity_dict
+
+# دالة لتقسيم النص إلى أجزاء بناءً على عدد التوكنز
+def split_text_into_chunks(text, tokenizer, max_length):
+    tokens = tokenizer.tokenize(text)
+    chunks = []
+    for i in range(0, len(tokens), max_length):
+        chunk_tokens = tokens[i:i + max_length]
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
+        chunks.append(chunk_text)
+    return chunks
+
+# دالة لتجزئة النص إلى جمل باستخدام التعبيرات العادية
+def extract_sentences(text):
+    sentences = re.split(r'(?<=[.!؟]) +', text)
+    return sentences
+
+# دالة لاستخراج الاقتباسات من النص
+def extract_quotes(text):
+    quotes = re.findall(r'[“"«](.*?)[”"»]', text)
+    return quotes
+
+# دالة لعد الرموز في النص
+def count_tokens(text, tokenizer):
+    tokens = tokenizer.tokenize(text)
+    return len(tokens)
+
+# دالة لاستخراج النص من ملفات PDF
+def extract_pdf_text(file_path):
+    with open(file_path, "rb") as pdf_file:
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        text = ""
+        for page_num in range(len(pdf_reader.pages)):
+            page = pdf_reader.pages[page_num]
+            text += page.extract_text()
+    return text
+
+# دالة لاستخراج النص من ملفات DOCX
+def extract_docx_text(file_path):
+    doc = docx.Document(file_path)
+    text = "\n".join([para.text for para in doc.paragraphs])
+    return text
+
+# دالة لقراءة النص من ملف مع التعامل مع مشاكل الترميز
+def read_text_file(file_path):
+    try:
+        with open(file_path, "r", encoding="utf-8") as file:
+            return file.read()
+    except UnicodeDecodeError:
+        try:
+            with open(file_path, "r", encoding="latin-1") as file:
+                return file.read()
+        except UnicodeDecodeError:
+            with open(file_path, "r", encoding="cp1252") as file:
+                return file.read()
+
+# دالة لاستخراج المشاهد من النص
+def extract_scenes(text):
+    scenes = re.split(r'داخلي|خارجي', text)
+    scenes = [scene.strip() for scene in scenes if scene.strip()]
+    return scenes
+
+# دالة لاستخراج تفاصيل المشهد (المكان والوقت)
+def extract_scene_details(scene):
+    details = {}
+    location_match = re.search(r'(داخلي|خارجي)', scene)
+    time_match = re.search(r'(ليلاً|نهاراً|شروق|غروب)', scene)
+
+    if location_match:
+        details['location'] = location_match.group()
+    if time_match:
+        details['time'] = time_match.group()
+
+    return details
+
+# دالة لاستخراج أعمار الشخصيات
+def extract_ages(text):
+    ages = re.findall(r'\b(\d{1,2})\s*(?:عام|سنة|سنوات)\s*(?:من العمر|عمره|عمرها)', text)
+    return ages
+
+# دالة لاستخراج وصف الشخصيات
+def extract_character_descriptions(text):
+    descriptions = re.findall(r'شخصية\s*(.*?)\s*:\s*وصف\s*(.*?)\s*(?:\.|،)', text, re.DOTALL)
+    return descriptions
+
+# دالة لاستخراج تكرار الشخصيات
+def extract_character_frequency(entities):
+    persons = [ent[0] for ent in entities['PERSON']]
+    frequency = Counter(persons)
+    return frequency
+
+# دالة لاستخراج الحوارات وتحديد المتحدثين
+def extract_dialogues(text):
+    dialogues = re.findall(r'(.*?)(?:\s*:\s*)(.*?)(?=\n|$)', text, re.DOTALL)
+    return dialogues
+
+# دالة لمعالجة الملفات وتقسيمها بناءً على عدد التوكنز
+def process_files(input_directory, output_directory_500):
+    for file_name in os.listdir(input_directory):
+        file_path = os.path.join(input_directory, file_name)
+        
+        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
+            continue
+        
+        if file_path.endswith(".pdf"):
+            text = extract_pdf_text(file_path)
+        elif file_path.endswith(".docx"):
+            text = extract_docx_text(file_path)
+        else:
+            text = read_text_file(file_path)
+        
+        # تقسيم النص إلى أجزاء لا تتجاوز 450 توكنز
+        chunks_450 = split_text_into_chunks(text, arabic_bert_tokenizer, 450)
+        for i, chunk in enumerate(chunks_450):
+            output_file_450 = os.path.join(output_directory_500, f"{os.path.splitext(file_name)[0]}_part_{i+1}.txt")
+            with open(output_file_450, "w", encoding="utf-8") as file:
+                file.write(chunk)
+
+# دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
+def analyze_files(input_directory, output_directory, tokenizer, max_length):
+    for file_name in os.listdir(input_directory):
+        file_path = os.path.join(input_directory, file_name)
+        
+        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
+            continue
+        
+        with open(file_path, "r", encoding="utf-8") as file:
+            text = file.read()
+        
+        chunks = split_text_into_chunks(text, tokenizer, max_length)
+        
+        # إجراء التحليل على النصوص المقسمة
+        for chunk in chunks:
+            entities = camel_ner_analysis(chunk)
+            sentences = extract_sentences(chunk)
+            quotes = extract_quotes(chunk)
+            token_count = count_tokens(chunk, tokenizer)
+            scenes = extract_scenes(chunk)
+            ages = extract_ages(chunk)
+            character_descriptions = extract_character_descriptions(chunk)
+            character_frequency = extract_character_frequency(entities)
+            dialogues = extract_dialogues(chunk)
+            scene_details = [extract_scene_details(scene) for scene in scenes]
+
+            # حفظ النتائج
+            with open(os.path.join(output_directory, f"{file_name}_entities.txt"), "a", encoding="utf-8") as file:
+                file.write(str(entities))
+
+            with open(os.path.join(output_directory, f"{file_name}_sentences.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(sentences))
+
+            with open(os.path.join(output_directory, f"{file_name}_quotes.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(quotes))
+
+            with open(os.path.join(output_directory, f"{file_name}_token_count.txt"), "a", encoding="utf-8") as file:
+                file.write(str(token_count))
+
+            with open(os.path.join(output_directory, f"{file_name}_scenes.txt"), "a", encoding="utf-8") as file:
+                file.write("\n".join(scenes))
+
+            with open(os.path.join(output_directory, f"{file_name}_scene_details.txt"), "a", encoding="utf-8") as file:
+                file.write(str(scene_details))
+
+            with open(os.path.join(output_directory, f"{file_name}_ages.txt"), "a", encoding="utf-8") as file:
+                file.write(str(ages))
+
+            with open(os.path.join(output_directory, f"{file_name}_character_descriptions.txt"), "a", encoding="utf-8") as file:
+                file.write(str(character_descriptions))
+
+            with open(os.path.join(output_directory, f"{file_name}_character_frequency.txt"), "a", encoding="utf-8") as file:
+                file.write(str(character_frequency))
+
+            with open(os.path.join(output_directory, f"{file_name}_dialogues.txt"), "a", encoding="utf-8") as file:
+                file.write(str(dialogues))
+
+# تحديد المسارات
+input_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/in"
+output_directory_450 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/500"
+input_directory_450 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/500"
+output_directory_450_out = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out/500"
+
+# التأكد من وجود المسارات
+os.makedirs(output_directory_450, exist_ok=True)
+os.makedirs(output_directory_450_out, exist_ok=True)
+
+# معالجة الملفات وتقسيمها
+process_files(input_directory, output_directory_450)
+
+# تحليل الملفات المقسمة إلى 450 توكنز
+analyze_files(input_directory_450, output_directory_450_out, arabic_bert_tokenizer, 512)
+
+print("تمت معالجة الملفات وتحليلها بنجاح.")