Delete firstkha.py

firstkha.py

@@ -1,231 +0,0 @@
-import os
-import re
-import torch
-from collections import Counter
-from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForTokenClassification
-import PyPDF2
-import openai
-import docx
-
-
-# التحقق من توفر GPU واستخدامه
-device = 0 if torch.cuda.is_available() else -1
-
-# تحميل نماذج BERT، GPT2، ELECTRA، و AraBERT
-arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
-arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")
-
-arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base")
-arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base")
-
-arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator")
-arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator")
-
-arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")
-
-# تحميل نموذج التعرف على الكيانات المسماة من CAMeL-Lab
-ner_tokenizer = AutoTokenizer.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
-ner_model = AutoModelForTokenClassification.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
-nlp_ner = pipeline("ner", model=ner_model, tokenizer=ner_tokenizer)
-
-# دالة لتحليل النص باستخدام transformers
-def camel_ner_analysis(text):
-    ner_results = nlp_ner(text)
-    entity_dict = {"PERSON": [], "LOC": [], "ORG": [], "DATE": []}
-    for entity in ner_results:
-        entity_type = entity["entity"]
-        if entity_type in entity_dict:
-            entity_dict[entity_type].append((entity["word"], entity_type))
-    return entity_dict
-
-# دالة لتقسيم النص إلى أجزاء بناءً على عدد التوكنز
-def split_text_into_chunks(text, tokenizer, max_length):
-    tokens = tokenizer.tokenize(text)
-    chunks = []
-    for i in range(0, len(tokens), max_length):
-        chunk_tokens = tokens[i:i + max_length]
-        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
-        chunks.append(chunk_text)
-    return chunks
-
-# دالة لتجزئة النص إلى جمل باستخدام التعبيرات العادية
-def extract_sentences(text):
-    sentences = re.split(r'(?<=[.!؟]) +', text)
-    return sentences
-
-# دالة لاستخراج الاقتباسات من النص
-def extract_quotes(text):
-    quotes = re.findall(r'[“"«](.*?)[”"»]', text)
-    return quotes
-
-# دالة لعد الرموز في النص
-def count_tokens(text, tokenizer):
-    tokens = tokenizer.tokenize(text)
-    return len(tokens)
-
-# دالة لاستخراج النص من ملفات PDF
-def extract_pdf_text(file_path):
-    with open(file_path, "rb") as pdf_file:
-        pdf_reader = PyPDF2.PdfReader(pdf_file)
-        text = ""
-        for page_num in range(len(pdf_reader.pages)):
-            page = pdf_reader.pages[page_num]
-            text += page.extract_text()
-    return text
-
-# دالة لاستخراج النص من ملفات DOCX
-def extract_docx_text(file_path):
-    doc = docx.Document(file_path)
-    text = "\n".join([para.text for para in doc.paragraphs])
-    return text
-
-# دالة لقراءة النص من ملف مع التعامل مع مشاكل الترميز
-def read_text_file(file_path):
-    try:
-        with open(file_path, "r", encoding="utf-8") as file:
-            return file.read()
-    except UnicodeDecodeError:
-        try:
-            with open(file_path, "r", encoding="latin-1") as file:
-                return file.read()
-        except UnicodeDecodeError:
-            with open(file_path, "r", encoding="cp1252") as file:
-                return file.read()
-
-# دالة لاستخراج المشاهد من النص
-def extract_scenes(text):
-    scenes = re.split(r'داخلي|خارجي', text)
-    scenes = [scene.strip() for scene in scenes if scene.strip()]
-    return scenes
-
-# دالة لاستخراج تفاصيل المشهد (المكان والوقت)
-def extract_scene_details(scene):
-    details = {}
-    location_match = re.search(r'(داخلي|خارجي)', scene)
-    time_match = re.search(r'(ليلاً|نهاراً|شروق|غروب)', scene)
-
-    if location_match:
-        details['location'] = location_match.group()
-    if time_match:
-        details['time'] = time_match.group()
-
-    return details
-
-# دالة لاستخراج أعمار الشخصيات
-def extract_ages(text):
-    ages = re.findall(r'\b(\d{1,2})\s*(?:عام|سنة|سنوات)\s*(?:من العمر|عمره|عمرها)', text)
-    return ages
-
-# دالة لاستخراج وصف الشخصيات
-def extract_character_descriptions(text):
-    descriptions = re.findall(r'شخصية\s*(.*?)\s*:\s*وصف\s*(.*?)\s*(?:\.|،)', text, re.DOTALL)
-    return descriptions
-
-# دالة لاستخراج تكرار الشخصيات
-def extract_character_frequency(entities):
-    persons = [ent[0] for ent in entities['PERSON']]
-    frequency = Counter(persons)
-    return frequency
-
-# دالة لاستخراج الحوارات وتحديد المتحدثين
-def extract_dialogues(text):
-    dialogues = re.findall(r'(.*?)(?:\s*:\s*)(.*?)(?=\n|$)', text, re.DOTALL)
-    return dialogues
-
-# دالة لمعالجة الملفات وتقسيمها بناءً على عدد التوكنز
-def process_files(input_directory, output_directory_500):
-    for file_name in os.listdir(input_directory):
-        file_path = os.path.join(input_directory, file_name)
-        
-        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
-            continue
-        
-        if file_path.endswith(".pdf"):
-            text = extract_pdf_text(file_path)
-        elif file_path.endswith(".docx"):
-            text = extract_docx_text(file_path)
-        else:
-            text = read_text_file(file_path)
-        
-        # تقسيم النص إلى أجزاء لا تتجاوز 450 توكنز
-        chunks_450 = split_text_into_chunks(text, arabic_bert_tokenizer, 450)
-        for i, chunk in enumerate(chunks_450):
-            output_file_450 = os.path.join(output_directory_500, f"{os.path.splitext(file_name)[0]}_part_{i+1}.txt")
-            with open(output_file_450, "w", encoding="utf-8") as file:
-                file.write(chunk)
-
-# دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
-def analyze_files(input_directory, output_directory, tokenizer, max_length):
-    for file_name in os.listdir(input_directory):
-        file_path = os.path.join(input_directory, file_name)
-        
-        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
-            continue
-        
-        with open(file_path, "r", encoding="utf-8") as file:
-            text = file.read()
-        
-        chunks = split_text_into_chunks(text, tokenizer, max_length)
-        
-        # إجراء التحليل على النصوص المقسمة
-        for chunk in chunks:
-            entities = camel_ner_analysis(chunk)
-            sentences = extract_sentences(chunk)
-            quotes = extract_quotes(chunk)
-            token_count = count_tokens(chunk, tokenizer)
-            scenes = extract_scenes(chunk)
-            ages = extract_ages(chunk)
-            character_descriptions = extract_character_descriptions(chunk)
-            character_frequency = extract_character_frequency(entities)
-            dialogues = extract_dialogues(chunk)
-            scene_details = [extract_scene_details(scene) for scene in scenes]
-
-            # حفظ النتائج
-            with open(os.path.join(output_directory, f"{file_name}_entities.txt"), "a", encoding="utf-8") as file:
-                file.write(str(entities))
-
-            with open(os.path.join(output_directory, f"{file_name}_sentences.txt"), "a", encoding="utf-8") as file:
-                file.write("\n".join(sentences))
-
-            with open(os.path.join(output_directory, f"{file_name}_quotes.txt"), "a", encoding="utf-8") as file:
-                file.write("\n".join(quotes))
-
-            with open(os.path.join(output_directory, f"{file_name}_token_count.txt"), "a", encoding="utf-8") as file:
-                file.write(str(token_count))
-
-            with open(os.path.join(output_directory, f"{file_name}_scenes.txt"), "a", encoding="utf-8") as file:
-                file.write("\n".join(scenes))
-
-            with open(os.path.join(output_directory, f"{file_name}_scene_details.txt"), "a", encoding="utf-8") as file:
-                file.write(str(scene_details))
-
-            with open(os.path.join(output_directory, f"{file_name}_ages.txt"), "a", encoding="utf-8") as file:
-                file.write(str(ages))
-
-            with open(os.path.join(output_directory, f"{file_name}_character_descriptions.txt"), "a", encoding="utf-8") as file:
-                file.write(str(character_descriptions))
-
-            with open(os.path.join(output_directory, f"{file_name}_character_frequency.txt"), "a", encoding="utf-8") as file:
-                file.write(str(character_frequency))
-
-            with open(os.path.join(output_directory, f"{file_name}_dialogues.txt"), "a", encoding="utf-8") as file:
-                file.write(str(dialogues))
-
-# تحديد المسارات
-input_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/in"
-output_directory_450 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/500"
-input_directory_450 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/500"
-output_directory_450_out = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out/500"
-
-# التأكد من وجود المسارات
-os.makedirs(output_directory_450, exist_ok=True)
-os.makedirs(output_directory_450_out, exist_ok=True)
-
-# معالجة الملفات وتقسيمها
-process_files(input_directory, output_directory_450)
-
-# تحليل الملفات المقسمة إلى 450 توكنز
-analyze_files(input_directory_450, output_directory_450_out, arabic_bert_tokenizer, 512)
-
-print("تمت معالجة الملفات وتحليلها بنجاح.")