Upload arabicpoetrygeneration.py

arabicpoetrygeneration.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+"""ArabicPoetryGeneration.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1HDyT5F8qnrbR_PW_HYpiM3O-7i6htGG2
+"""
+
+!pip install transformers
+!pip install tashaphyne
+!pip install gradio
+!pip install translate
+
+import pandas as pd
+import nltk
+from nltk.tokenize import word_tokenize
+from transformers import BertTokenizer
+from transformers import AutoTokenizer
+import random
+from tashaphyne import normalize
+import re
+import numpy as np
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional, GRU
+import tensorflow as tf
+from transformers import AutoTokenizer
+
+nltk.download('punkt')
+nltk.download('wordnet')
+
+aurl = 'https://raw.githubusercontent.com/Obai33/NLP_PoemGenerationDatasets/main/arabicpoems.csv'
+adf = pd.read_csv(aurl)
+
+# Function to normalize text
+def normalize_text(text):
+    normalize.strip_tashkeel(text)
+    normalize.strip_tatweel(text)
+    normalize.normalize_hamza(text)
+    normalize.normalize_lamalef(text)
+    return text
+
+# Normalize the text
+allah = normalize_text('الله')
+adf = adf['poem_text']
+i = random.randint(0, len(adf))
+adf = adf.sample(n=100, random_state=i)
+adf = adf.apply(lambda x: normalize_text(x))
+adf = adf[~adf.str.contains(allah)]
+
+# Function to clean text
+def remove_non_arabic_symbols(text):
+    arabic_pattern = r'[\u0600-\u06FF\s]+'
+    arabic_text = re.findall(arabic_pattern, text)
+    cleaned_text = ''.join(arabic_text)
+    return cleaned_text
+
+# Clean the text
+adf = adf.apply(lambda x: remove_non_arabic_symbols(x))
+
+# Tokenize the text
+tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
+tokens = tokenizer.tokenize(adf.tolist(), is_split_into_words=True)
+
+input_sequences = []
+for line in adf:
+    token_list = tokenizer.encode(line, add_special_tokens=True)
+    for i in range(1, len(token_list)):
+        n_gram_sequence = token_list[:i+1]
+        input_sequences.append(n_gram_sequence)
+
+max_sequence_len = 100
+input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))
+
+total_words = tokenizer.vocab_size
+
+xs, labels = input_sequences[:, :-1], input_sequences[:, -1]
+ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)
+
+
+##############
+
+import requests
+
+# URL of the model
+url = 'https://github.com/Obai33/NLP_PoemGenerationDatasets/raw/main/modelarab1.h5'
+# Local file path to save the model
+local_filename = 'modelarab1.h5'
+
+# Download the model file
+response = requests.get(url)
+with open(local_filename, 'wb') as f:
+    f.write(response.content)
+
+# Load the pre-trained model
+model = tf.keras.models.load_model(local_filename)
+
+##############
+
+# Import the necessary library for translation
+import translate
+
+# Function to translate text to English
+def translate_to_english(text):
+    translator = translate.Translator(from_lang="ar", to_lang="en")
+    translated_text = translator.translate(text)
+    return translated_text
+
+def generate_arabic_text(seed_text, next_words=50):
+    generated_text = seed_text
+    for _ in range(next_words):
+        token_list = tokenizer.encode(generated_text, add_special_tokens=False)
+        token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
+        predicted = np.argmax(model.predict(token_list), axis=-1)
+        output_word = tokenizer.decode(predicted[0])
+        generated_text += " " + output_word
+    reconnected_text = generated_text.replace(" ##", "")
+    t_text = translate_to_english(reconnected_text)
+    return reconnected_text, t_text
+
+import gradio as gr
+
+# Update Gradio interface to include both Arabic and English outputs
+iface = gr.Interface(
+    fn=generate_arabic_text,
+    inputs="text",
+    outputs=["text", "text"],
+    title="Arabic Poetry Generation",
+    description="Enter Arabic text to generate a small poem.",
+    theme="compact"
+)
+# Run the interface
+iface.launch()