pages/Workflow & Model Overview.py

import streamlit as st
import pandas as pd

# Custom CSS for better styling
st.markdown("""

    <style>

        .main-title {

            font-size: 36px;

            color: #4A90E2;

            font-weight: bold;

            text-align: center;

        }

        .sub-title {

            font-size: 24px;

            color: #4A90E2;

            margin-top: 20px;

        }

        .section {

            background-color: #f9f9f9;

            padding: 15px;

            border-radius: 10px;

            margin-top: 20px;

        }

        .section h2 {

            font-size: 22px;

            color: #4A90E2;

        }

        .section p, .section ul {

            color: #666666;

        }

        .link {

            color: #4A90E2;

            text-decoration: none;

        }

    </style>

""", unsafe_allow_html=True)

# Main Title
st.markdown('<div class="main-title">Arabic Named Entity Recognition - BERT-based Model</div>', unsafe_allow_html=True)

# Introduction
st.markdown("""

<div class="section">

    <p>Named Entity Recognition (NER) models identify and categorize important entities in a text. This page details a BERT-based NER model for Arabic texts, including Modern Standard Arabic (MSA), Dialectal Arabic (DA), and Classical Arabic (CA). The model is pretrained and available on Hugging Face, then imported into Spark NLP.</p>

</div>

""", unsafe_allow_html=True)

# Model Description
st.markdown('<div class="sub-title">Description</div>', unsafe_allow_html=True)
st.markdown("""

<div class="section">

    <p>The <code>bert_ner_bert_base_arabic_camelbert_mix_ner</code> model is pretrained for Arabic named entity recognition, originally trained by CAMeL-Lab. It can identify the following types of entities:</p>

    <ul>

        <li>ORG (Organization)</li>

        <li>LOC (Location)</li>

        <li>PERS (Person)</li>

        <li>MISC (Miscellaneous)</li>

    </ul>

</div>

""", unsafe_allow_html=True)

# Setup Instructions
st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
st.markdown('<p>To use the model, you need Spark NLP installed. You can install it using pip:</p>', unsafe_allow_html=True)
st.code("""

pip install spark-nlp

pip install pyspark

""", language="bash")

st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
st.code("""

import sparknlp



# Start Spark Session

spark = sparknlp.start()

""", language='python')

# Example Usage
st.markdown('<div class="sub-title">Example Usage with Arabic NER Model</div>', unsafe_allow_html=True)
st.markdown("""

<div class="section">

    <p>Below is an example of how to set up and use the <code>bert_ner_bert_base_arabic_camelbert_mix_ner</code> model for named entity recognition in Arabic:</p>

</div>

""", unsafe_allow_html=True)
st.code('''

from sparknlp.base import *

from sparknlp.annotator import *

from pyspark.ml import Pipeline

from pyspark.sql.functions import col, expr, round, concat, lit, explode



# Define the components of the pipeline

documentAssembler = DocumentAssembler() \\

    .setInputCol("text") \\

    .setOutputCol("document")



sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") \\

    .setInputCols(["document"]) \\

    .setOutputCol("sentence")



tokenizer = Tokenizer() \\

    .setInputCols(["sentence"]) \\

    .setOutputCol("token")



tokenClassifier = BertForTokenClassification.pretrained("bert_ner_bert_base_arabic_camelbert_mix_ner", "ar") \\

    .setInputCols(["sentence", "token"]) \\

    .setOutputCol("ner")



ner_converter = NerConverter()\\

    .setInputCols(["document", "token", "ner"])\\

    .setOutputCol("ner_chunk")



# Create the pipeline

pipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter])



# Create sample data

example = """

كانت مدينة بغداد، العاصمة الحالية للعراق، مركزاً ثقافياً وحضارياً عظيماً في العصور الوسطى. تأسست في القرن الثامن الميلادي على يد الخليفة العباسي أبو جعفر المنصور. 

كانت بغداد مدينة المعرفة والعلم، حيث توافد إليها العلماء والفلاسفة من كل أنحاء العالم الإسلامي للدراسة في بيت الحكمة. كانت مكتباتها تحتوي على آلاف المخطوطات النادرة، 

وكانت تشتهر بمدارسها العلمية والطبية والفلكية. في عام 1258، سقطت بغداد في يد المغول بقيادة هولاكو خان، مما أدى إلى تدمير جزء كبير من المدينة وخسارة العديد من النفائس.

"""

data = spark.createDataFrame([[example]]).toDF("text")



# Fit and transform data with the pipeline

result = pipeline.fit(data).transform(data)



# Select the result, entity

result.select(

    expr("explode(ner_chunk) as ner_chunk")

).select(

    col("ner_chunk.result").alias("chunk"),

    col("ner_chunk.metadata").getItem("entity").alias("ner_label")

).show(truncate=False)

''', language="python")

# Data for the DataFrame
data = {
    "chunk": ["جعفر المنصور", "بغداد", "بغداد", "هولاكو"],
    "ner_label": ["PERS", "LOC", "LOC", "PERS"]
}

# Creating the DataFrame
df = pd.DataFrame(data)
df.index += 1
st.dataframe(df)

# Model Information
st.markdown('<div class="sub-title">Model Information</div>', unsafe_allow_html=True)
st.markdown("""

<div class="section">

    <p>The <code>bert_ner_bert_base_arabic_camelbert_mix_ner</code> model details are as follows:</p>

    <ul>

        <li><strong>Model Name:</strong> bert_ner_bert_base_arabic_camelbert_mix_ner</li>

        <li><strong>Compatibility:</strong> Spark NLP 3.4.2+</li>

        <li><strong>License:</strong> Open Source</li>

        <li><strong>Edition:</strong> Official</li>

        <li><strong>Input Labels:</strong> [document, token]</li>

        <li><strong>Output Labels:</strong> [ner]</li>

        <li><strong>Language:</strong> ar</li>

        <li><strong>Size:</strong> 407.3 MB</li>

        <li><strong>Case sensitive:</strong> true</li>

        <li><strong>Max sentence length:</strong> 128</li>

    </ul>

</div>

""", unsafe_allow_html=True)

# Summary
st.markdown('<div class="sub-title">Summary</div>', unsafe_allow_html=True)
st.markdown("""

<div class="section">

    <p>This page provided an overview of the <code>bert_ner_bert_base_arabic_camelbert_mix_ner</code> model for Arabic NER. We discussed how to set up and use the model with Spark NLP, including example code and results. We also provided details on the model's specifications and links to relevant resources for further exploration.</p>

</div>

""", unsafe_allow_html=True)

# References
st.markdown('<div class="sub-title">Model References</div>', unsafe_allow_html=True)
st.markdown("""

<div class="section">

    <ul>

        <li><a class="link" href="https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-mix-ner" target="_blank" rel="noopener">Hugging Face Model Page</a></li>

        <li><a class="link" href="https://camel.abudhabi.nyu.edu/anercorp/" target="_blank" rel="noopener">ANERcorp</a></li>

        <li><a class="link" href="https://arxiv.org/abs/2103.06678" target="_blank" rel="noopener">Research Paper</a></li>

        <li><a class="link" href="https://github.com/CAMeL-Lab/CAMeLBERT" target="_blank" rel="noopener">CAMeLBERT GitHub</a></li>

        <li><a class="link" href="https://github.com/CAMeL-Lab/camel_tools" target="_blank" rel="noopener">CAMeL Tools GitHub</a></li>

    </ul>

</div>

""", unsafe_allow_html=True)

# Community & Support
st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
st.markdown("""

<div class="section">

    <ul>

        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>

        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub Repository</a>: Report issues or contribute</li>

        <li><a class="link" href="https://forum.johnsnowlabs.com/" target="_blank">Community Forum</a>: Ask questions, share ideas, and get support</li>

    </ul>

</div>

""", unsafe_allow_html=True)