firstkha.py

import os
import re
import torch
from collections import Counter
from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForTokenClassification
import PyPDF2
import openai
import docx


# التحقق من توفر GPU واستخدامه
device = 0 if torch.cuda.is_available() else -1

# تحميل نماذج BERT، GPT2، ELECTRA، و AraBERT
arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")

arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base")
arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base")

arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator")
arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator")

arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")

# تحميل نموذج التعرف على الكيانات المسماة من CAMeL-Lab
ner_tokenizer = AutoTokenizer.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
ner_model = AutoModelForTokenClassification.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
nlp_ner = pipeline("ner", model=ner_model, tokenizer=ner_tokenizer)

# دالة لتحليل النص باستخدام transformers
def camel_ner_analysis(text):
    ner_results = nlp_ner(text)
    entity_dict = {"PERSON": [], "LOC": [], "ORG": [], "DATE": []}
    for entity in ner_results:
        entity_type = entity["entity"]
        if entity_type in entity_dict:
            entity_dict[entity_type].append((entity["word"], entity_type))
    return entity_dict

# دالة لتقسيم النص إلى أجزاء بناءً على عدد التوكنز
def split_text_into_chunks(text, tokenizer, max_length):
    tokens = tokenizer.tokenize(text)
    chunks = []
    for i in range(0, len(tokens), max_length):
        chunk_tokens = tokens[i:i + max_length]
        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
        chunks.append(chunk_text)
    return chunks

# دالة لتجزئة النص إلى جمل باستخدام التعبيرات العادية
def extract_sentences(text):
    sentences = re.split(r'(?<=[.!؟]) +', text)
    return sentences

# دالة لاستخراج الاقتباسات من النص
def extract_quotes(text):
    quotes = re.findall(r'[“"«](.*?)[”"»]', text)
    return quotes

# دالة لعد الرموز في النص
def count_tokens(text, tokenizer):
    tokens = tokenizer.tokenize(text)
    return len(tokens)

# دالة لاستخراج النص من ملفات PDF
def extract_pdf_text(file_path):
    with open(file_path, "rb") as pdf_file:
        pdf_reader = PyPDF2.PdfReader(pdf_file)
        text = ""
        for page_num in range(len(pdf_reader.pages)):
            page = pdf_reader.pages[page_num]
            text += page.extract_text()
    return text

# دالة لاستخراج النص من ملفات DOCX
def extract_docx_text(file_path):
    doc = docx.Document(file_path)
    text = "\n".join([para.text for para in doc.paragraphs])
    return text

# دالة لقراءة النص من ملف مع التعامل مع مشاكل الترميز
def read_text_file(file_path):
    try:
        with open(file_path, "r", encoding="utf-8") as file:
            return file.read()
    except UnicodeDecodeError:
        try:
            with open(file_path, "r", encoding="latin-1") as file:
                return file.read()
        except UnicodeDecodeError:
            with open(file_path, "r", encoding="cp1252") as file:
                return file.read()

# دالة لاستخراج المشاهد من النص
def extract_scenes(text):
    scenes = re.split(r'داخلي|خارجي', text)
    scenes = [scene.strip() for scene in scenes if scene.strip()]
    return scenes

# دالة لاستخراج تفاصيل المشهد (المكان والوقت)
def extract_scene_details(scene):
    details = {}
    location_match = re.search(r'(داخلي|خارجي)', scene)
    time_match = re.search(r'(ليلاً|نهاراً|شروق|غروب)', scene)

    if location_match:
        details['location'] = location_match.group()
    if time_match:
        details['time'] = time_match.group()

    return details

# دالة لاستخراج أعمار الشخصيات
def extract_ages(text):
    ages = re.findall(r'\b(\d{1,2})\s*(?:عام|سنة|سنوات)\s*(?:من العمر|عمره|عمرها)', text)
    return ages

# دالة لاستخراج وصف الشخصيات
def extract_character_descriptions(text):
    descriptions = re.findall(r'شخصية\s*(.*?)\s*:\s*وصف\s*(.*?)\s*(?:\.|،)', text, re.DOTALL)
    return descriptions

# دالة لاستخراج تكرار الشخصيات
def extract_character_frequency(entities):
    persons = [ent[0] for ent in entities['PERSON']]
    frequency = Counter(persons)
    return frequency

# دالة لاستخراج الحوارات وتحديد المتحدثين
def extract_dialogues(text):
    dialogues = re.findall(r'(.*?)(?:\s*:\s*)(.*?)(?=\n|$)', text, re.DOTALL)
    return dialogues

# دالة لمعالجة الملفات وتقسيمها بناءً على عدد التوكنز
def process_files(input_directory, output_directory_500):
    for file_name in os.listdir(input_directory):
        file_path = os.path.join(input_directory, file_name)
        
        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
            continue
        
        if file_path.endswith(".pdf"):
            text = extract_pdf_text(file_path)
        elif file_path.endswith(".docx"):
            text = extract_docx_text(file_path)
        else:
            text = read_text_file(file_path)
        
        # تقسيم النص إلى أجزاء لا تتجاوز 450 توكنز
        chunks_450 = split_text_into_chunks(text, arabic_bert_tokenizer, 450)
        for i, chunk in enumerate(chunks_450):
            output_file_450 = os.path.join(output_directory_500, f"{os.path.splitext(file_name)[0]}_part_{i+1}.txt")
            with open(output_file_450, "w", encoding="utf-8") as file:
                file.write(chunk)

# دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
def analyze_files(input_directory, output_directory, tokenizer, max_length):
    for file_name in os.listdir(input_directory):
        file_path = os.path.join(input_directory, file_name)
        
        if os.path.isdir(file_path):  # التأكد من أن الملف ليس مجلدًا
            continue
        
        with open(file_path, "r", encoding="utf-8") as file:
            text = file.read()
        
        chunks = split_text_into_chunks(text, tokenizer, max_length)
        
        # إجراء التحليل على النصوص المقسمة
        for chunk in chunks:
            entities = camel_ner_analysis(chunk)
            sentences = extract_sentences(chunk)
            quotes = extract_quotes(chunk)
            token_count = count_tokens(chunk, tokenizer)
            scenes = extract_scenes(chunk)
            ages = extract_ages(chunk)
            character_descriptions = extract_character_descriptions(chunk)
            character_frequency = extract_character_frequency(entities)
            dialogues = extract_dialogues(chunk)
            scene_details = [extract_scene_details(scene) for scene in scenes]

            # حفظ النتائج
            with open(os.path.join(output_directory, f"{file_name}_entities.txt"), "a", encoding="utf-8") as file:
                file.write(str(entities))

            with open(os.path.join(output_directory, f"{file_name}_sentences.txt"), "a", encoding="utf-8") as file:
                file.write("\n".join(sentences))

            with open(os.path.join(output_directory, f"{file_name}_quotes.txt"), "a", encoding="utf-8") as file:
                file.write("\n".join(quotes))

            with open(os.path.join(output_directory, f"{file_name}_token_count.txt"), "a", encoding="utf-8") as file:
                file.write(str(token_count))

            with open(os.path.join(output_directory, f"{file_name}_scenes.txt"), "a", encoding="utf-8") as file:
                file.write("\n".join(scenes))

            with open(os.path.join(output_directory, f"{file_name}_scene_details.txt"), "a", encoding="utf-8") as file:
                file.write(str(scene_details))

            with open(os.path.join(output_directory, f"{file_name}_ages.txt"), "a", encoding="utf-8") as file:
                file.write(str(ages))

            with open(os.path.join(output_directory, f"{file_name}_character_descriptions.txt"), "a", encoding="utf-8") as file:
                file.write(str(character_descriptions))

            with open(os.path.join(output_directory, f"{file_name}_character_frequency.txt"), "a", encoding="utf-8") as file:
                file.write(str(character_frequency))

            with open(os.path.join(output_directory, f"{file_name}_dialogues.txt"), "a", encoding="utf-8") as file:
                file.write(str(dialogues))

# تحديد المسارات
input_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/in"
output_directory_450 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/500"
input_directory_450 = "/Volumes/CLOCKWORK T/clockworkspace/first pro/500"
output_directory_450_out = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out/500"

# التأكد من وجود المسارات
os.makedirs(output_directory_450, exist_ok=True)
os.makedirs(output_directory_450_out, exist_ok=True)

# معالجة الملفات وتقسيمها
process_files(input_directory, output_directory_450)

# تحليل الملفات المقسمة إلى 450 توكنز
analyze_files(input_directory_450, output_directory_450_out, arabic_bert_tokenizer, 512)

print("تمت معالجة الملفات وتحليلها بنجاح.")