Upload 3 files

app.py

@@ -0,0 +1,55 @@
+from urllib.parse import unquote
+
+import arabic_reshaper
+import streamlit as st
+from bidi.algorithm import get_display
+from summarize import get_results
+
+
+st.set_page_config(
+    page_title="Arabic Summarization",
+    page_icon="🤖",
+    layout="wide",
+    initial_sidebar_state="expanded",
+    menu_items={
+         'Get Help': 'https://www.extremelycoolapp.com/help',
+         'Report a bug': "https://www.extremelycoolapp.com/bug",
+         'About': "# Arabic Text Summarizeation , abdalrahman shahrour",
+    }
+)
+
+rtl = lambda w: get_display(f"{arabic_reshaper.reshape(w)}")
+
+st.header('Arabic Text Summarization')
+
+
+st.markdown("")
+
+model = st.sidebar.selectbox('Select one', ['arabartsummarization', 'AraBART', 'auto-arabic-summarization', 'BERT2BERT', 'xlmroberta2xlmroberta', 'nltk_summarizer'],help="Model",)
+        # ahmeddbahaa/xlmroberta2xlmroberta-finetune-summarization-ar
+st.sidebar.write("\n")
+num_beams = st.sidebar.slider(
+    "Number of beams", min_value=1, max_value=10, value=3, step=1
+)
+
+st.sidebar.write("\n")
+length_penalty = st.sidebar.slider(
+        "Length penalty ", min_value=0.1, max_value=3.0, value=1.0, step=0.1,
+)
+number_of_sentence = st.sidebar.slider(
+    "Number of sentence", min_value=1, max_value=10, value=3, step=1
+)
+
+
+        
+doc = st.text_area("Enter the text to be summarized",height=550,value=" شهدت مدينة طرابلس، مساء أمس الأربعاء، احتجاجات شعبية وأعمال شغب لليوم الثالث على التوالي، وذلك بسبب تردي الوضع المعيشي والاقتصادي. واندلعت مواجهات عنيفة وعمليات كر وفر ما بين الجيش اللبناني والمحتجين استمرت لساعات، إثر محاولة فتح الطرقات المقطوعة، ما أدى إلى إصابة العشرات من الطرفين.")
+
+summarize_button = st.button(label="🧞‍♂️ summarize 🧞‍♂️")
+
+if summarize_button:
+    with st.spinner("جاري التلخيص ..."):
+        result = get_results(doc, model, num_beams, length_penalty,number_of_sentence)
+    if len(result) > 0:
+        st.write(result)
+    else:
+        st.write("")

requirements.txt

@@ -0,0 +1,70 @@
+altair==4.2.0
+arabert==1.0.1
+arabic-reshaper==2.1.3
+attrs==22.2.0
+blinker==1.5
+cachetools==5.2.0
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.3
+codetiming==1.3.0
+commonmark==0.9.1
+decorator==5.1.1
+emoji==1.4.2
+entrypoints==0.4
+farasapy==0.0.14
+filelock==3.8.2
+future==0.18.2
+gitdb==4.0.10
+GitPython==3.1.29
+huggingface-hub==0.11.1
+idna==3.4
+importlib-metadata==5.2.0
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+MarkupSafe==2.1.1
+nltk==3.8
+numpy==1.24.0
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+packaging==22.0
+pandas==1.5.2
+Pillow==9.3.0
+preprocess==2.0.0
+protobuf==3.20.2
+PyArabic==0.6.15
+pyarrow==10.0.1
+pydeck==0.8.0
+Pygments==2.13.0
+Pympler==1.0.1
+pyrsistent==0.19.2
+python-bidi==0.4.2
+python-dateutil==2.8.2
+pytz==2022.7
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+regex==2022.10.31
+requests==2.28.1
+rich==12.6.0
+semver==2.13.0
+sentencepiece==0.1.97
+six==1.16.0
+smmap==5.0.0
+streamlit==1.16.0
+tokenizers==0.13.2
+toml==0.10.2
+toolz==0.12.0
+torch==1.13.1
+tornado==6.2
+tqdm==4.64.1
+transformers==4.25.1
+typing_extensions==4.4.0
+tzdata==2022.7
+tzlocal==4.2
+urllib3==1.26.13
+validators==0.20.0
+watchdog==2.2.0
+zipp==3.11.0

summarize.py

@@ -0,0 +1,171 @@
+import logging
+import os
+import re
+from functools import lru_cache
+from urllib.parse import unquote
+
+import streamlit as st
+from codetiming import Timer
+from transformers import pipeline
+from arabert.preprocess import ArabertPreprocessor
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForCausalLM
+import tokenizers
+import re
+import heapq  
+from string import punctuation
+import nltk
+from nltk.corpus import stopwords
+
+punctuation = punctuation + '\n'
+logger = logging.getLogger(__name__)
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+logger.info("Loading models...")
+reader_time = Timer("loading", text="Time: {:.2f}", logger=logging.info)
+reader_time.start()
+
+
+reader_time.stop()
+
+
+logger.info("Finished loading the models...")
+logger.info(f"Time spent loading: {reader_time.last}")
+
+@lru_cache(maxsize=200)
+def get_results(text, model_selected, num_beams, length_penalty,number_of_sentence):
+    logger.info("\n=================================================================")
+    logger.info(f"Text: {text}")
+    logger.info(f"model_selected: {model_selected}")
+    logger.info(f"length_penalty: {length_penalty}")
+    reader_time = Timer("summarize", text="Time: {:.2f}", logger=logging.info)
+    reader_time.start()
+    if model_selected == 'GPT-2':
+        number_of_tokens_limit = 80
+    else:
+        number_of_tokens_limit = 150
+    logger.info(f"input length: {len(text.split())}")
+
+    if model_selected == 'arabartsummarization':
+        model_name="abdalrahmanshahrour/arabartsummarization"
+        preprocessor = ArabertPreprocessor(model_name="")
+
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
+        result = pipeline1(text,
+            pad_token_id= tokenizer.eos_token_id,
+            num_beams=num_beams,
+            repetition_penalty=3.0,
+            max_length=200,
+            length_penalty=length_penalty,
+            no_repeat_ngram_size = 3)[0]['generated_text']
+        logger.info('arabartsummarization')
+    elif model_selected == 'AraBART':
+
+        model_name= "abdalrahmanshahrour/AraBART-summ"
+        preprocessor = ArabertPreprocessor(model_name="")
+
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
+        result = pipeline1(text,
+            pad_token_id= tokenizer.eos_token_id,
+            num_beams=num_beams,
+            repetition_penalty=3.0,
+            max_length=200,
+            length_penalty=length_penalty,
+            no_repeat_ngram_size = 3)[0]['generated_text']
+        logger.info('AraBART')
+    
+    elif model_selected == "auto-arabic-summarization":
+
+        model_name="abdalrahmanshahrour/auto-arabic-summarization"
+        preprocessor = ArabertPreprocessor(model_name="")
+
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
+        result = pipeline1(text,
+            pad_token_id= tokenizer.eos_token_id,
+            num_beams=num_beams,
+            repetition_penalty=3.0,
+            max_length=200,
+            length_penalty=length_penalty,
+            no_repeat_ngram_size = 3)[0]['generated_text']
+        logger.info('auto-arabic-summarization')
+    
+    elif model_selected == 'BERT2BERT':
+
+        model_name="malmarjeh/bert2bert"
+        preprocessor = ArabertPreprocessor(model_name="")
+
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
+        result = pipeline1(text,
+            pad_token_id= tokenizer.eos_token_id,
+            num_beams=num_beams,
+            repetition_penalty=3.0,
+            max_length=200,
+            length_penalty=length_penalty,
+            no_repeat_ngram_size = 3)[0]['generated_text']
+        logger.info('BERT2BERT')
+    
+    elif model_selected == "xlmroberta2xlmroberta":
+        model_name="ahmeddbahaa/xlmroberta2xlmroberta-finetune-summarization-ar"
+        preprocessor = ArabertPreprocessor(model_name="")
+
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
+        result = pipeline1(text,
+            pad_token_id= tokenizer.eos_token_id,
+            num_beams=num_beams,
+            repetition_penalty=3.0,
+            max_length=200,
+            length_penalty=length_penalty,
+            no_repeat_ngram_size = 3)[0]['generated_text']
+        logger.info('xlmroberta2xlmroberta')
+    
+    elif model_selected == "nltk_summarizer":
+        # number_of_sentence = 3
+        stopWords = set(nltk.corpus.stopwords.words("arabic") + nltk.corpus.stopwords.words("english"))
+        word_frequencies = {}  
+        for word in nltk.word_tokenize(text):  
+            if word not in stopWords:
+                if word not in punctuation:
+                    if word not in word_frequencies.keys():
+                        word_frequencies[word] = 1
+                    else:
+                        word_frequencies[word] += 1
+
+        maximum_frequncy = max(list(word_frequencies.values()),default=3)
+
+        for word in word_frequencies.keys():  
+            word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)
+
+        sentence_list = nltk.sent_tokenize(text)
+        sentence_scores = {}
+        for sent in sentence_list:
+            for word in nltk.word_tokenize(sent.lower()):
+                if word in word_frequencies.keys():
+                    if len(sent.split(' ')) < 30:
+                        if sent not in sentence_scores.keys():
+                            sentence_scores[sent] = word_frequencies[word]
+                        else:
+                            sentence_scores[sent] += word_frequencies[word]
+
+        summary_sentences = heapq.nlargest(number_of_sentence, sentence_scores, key=sentence_scores.get)
+
+        result = ' '.join(summary_sentences)  
+    else:
+        result = "الرجاء اختيار نموذج"
+
+    reader_time.stop()
+    logger.info(f"Time spent summarizing: {reader_time.last}")
+
+    return result
+
+
+if __name__ == "__main__":
+    results_dict = ""