Upload 6 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo-old.py

@@ -0,0 +1,312 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+from streamlit_tags import st_tags
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model, task, zeroShotLables=['']):
+    document_assembler = DocumentAssembler() \
+        .setInputCol('text') \
+        .setOutputCol('document')
+
+    sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\
+        .setInputCols(["document"])\
+        .setOutputCol("sentence")
+
+    tokenizer = Tokenizer() \
+        .setInputCols(['sentence']) \
+        .setOutputCol('token')
+
+    if task == "Token Classification":
+        TCclassifier = DeBertaForTokenClassification \
+            .pretrained("deberta_v3_small_token_classifier_conll03", "en") \
+            .setInputCols(["sentence", "token"]) \
+            .setOutputCol("ner") \
+            .setCaseSensitive(True) \
+            .setMaxSentenceLength(512)
+        
+        ner_converter = NerConverter() \
+            .setInputCols(['sentence', 'token', 'ner']) \
+            .setOutputCol('ner_chunk')
+
+        TCpipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, TCclassifier, ner_converter])
+        return TCpipeline
+
+    elif task == "Zero-Shot Classification":
+        ZSCtokenizer = Tokenizer() \
+            .setInputCols(['document']) \
+            .setOutputCol('token')
+        
+        zeroShotClassifier = DeBertaForZeroShotClassification \
+            .pretrained('deberta_base_zero_shot_classifier_mnli_anli_v3', 'en') \
+            .setInputCols(['token', 'document']) \
+            .setOutputCol('class') \
+            .setCaseSensitive(False) \
+            .setMaxSentenceLength(512) \
+            .setCandidateLabels(zeroShotLables)
+
+        ZSCpipeline = Pipeline(stages=[document_assembler, ZSCtokenizer, zeroShotClassifier])
+        return ZSCpipeline
+
+    elif task == "Sequence Classification":
+        SCtokenizer = Tokenizer() \
+            .setInputCols(['document']) \
+            .setOutputCol('token')
+
+        sequence_classifier = DeBertaForSequenceClassification \
+            .pretrained("deberta_v3_base_sequence_classifier_imdb", "en") \
+            .setInputCols(["document", "token"]) \
+            .setOutputCol("class")
+        
+        SCpipeline = Pipeline(stages=[document_assembler, SCtokenizer, sequence_classifier])
+        return SCpipeline
+
+    elif task == "Question Answering":
+        QAdocument_assembler = MultiDocumentAssembler()\
+            .setInputCols(["question", "context"]) \
+            .setOutputCols(["document_question", "document_context"])
+
+        spanClassifier = DebertaForQuestionAnswering \
+            .pretrained("deberta_v3_xsmall_qa_squad2", "en") \
+            .setInputCols(["document_question","document_context"]) \
+            .setOutputCol("answer")
+
+        QApipeline = Pipeline(stages=[QAdocument_assembler, spanClassifier])
+        return QApipeline
+
+def fit_data(pipeline, data, task, ques='', cont=''):
+    if task in ['Token Classification', 'Sequence Classification']:
+      empty_df = spark.createDataFrame([['']]).toDF('text')
+      pipeline_model = pipeline.fit(empty_df)
+      model = LightPipeline(pipeline_model)
+      result = model.fullAnnotate(data)
+      return result
+    else:
+      df = spark.createDataFrame([[ques, cont]]).toDF("question", "context")
+      result = pipeline.fit(df).transform(df)
+      return result.select('answer.result').collect()
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+tasks_models_descriptions = {
+    "Token Classification": {
+        "models": ["deberta_v3_small_token_classifier_conll03"],
+        "description": "The 'deberta_v3_small_token_classifier_conll03' model is adept at token classification tasks, including named entity recognition (NER). It identifies and categorizes tokens in text, such as names, dates, and locations, enhancing the extraction of meaningful information from unstructured data."
+    },
+    "Zero-Shot Classification": {
+        "models": ["deberta_base_zero_shot_classifier_mnli_anli_v3"],
+        "description": "The 'deberta_base_zero_shot_classifier_mnli_anli_v3' model provides flexible text classification without needing training data for specific categories. It is ideal for dynamic scenarios where text needs to be categorized into topics like urgent issues, technology, or sports without prior labeling."
+    },
+    "Sequence Classification": {
+        "models": ["deberta_v3_base_sequence_classifier_imdb"],
+        "description": "The 'deberta_v3_base_sequence_classifier_imdb' model is proficient in sequence classification tasks, such as sentiment analysis and document categorization. It effectively determines the sentiment of reviews, classifies text, and sorts documents based on their content and context."
+    },
+    "Question Answering": {
+        "models": ["deberta_v3_xsmall_qa_squad2"],
+        "description": "The 'deberta_v3_xsmall_qa_squad2' model, based on RoBERTa, is designed for precise question answering. They excel in extracting answers from a given context, making them suitable for developing advanced QA systems, enhancing customer support, and retrieving specific information from text."
+    }
+}
+
+# Sidebar content
+task = st.sidebar.selectbox("Choose the task", list(tasks_models_descriptions.keys()))
+model = st.sidebar.selectbox("Choose the pretrained model", tasks_models_descriptions[task]["models"], help="For more info about the models visit: https://sparknlp.org/models")
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/357691d18373d6e8f13b5b1015137a398fd0a45f/Spark_NLP_Udemy_MOOC/Open_Source/17.01.Transformers-based_Embeddings.ipynb#L103">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Page content
+title, sub_title = (f'DeBERTa for {task}', tasks_models_descriptions[task]["description"])
+st.markdown(f'<div class="main-title">{title}</div>', unsafe_allow_html=True)
+container = st.container(border=True)
+container.write(sub_title)
+
+# Load examples
+examples_mapping = {
+    "Token Classification": [
+        "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+        "The Mona Lisa is a 16th century oil painting created by Leonardo. It's held at the Louvre in Paris.",
+        "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously. “I can tell you very senior CEOs of major American car companies would shake my hand and turn away because I wasn’t worth talking to,” said Thrun, now the co-founder and CEO of online higher education startup Udacity, in an interview with Recode earlier this week.",
+        "Facebook is a social networking service launched as TheFacebook on February 4, 2004. It was founded by Mark Zuckerberg with his college roommates and fellow Harvard University students Eduardo Saverin, Andrew McCollum, Dustin Moskovitz and Chris Hughes. The website's membership was initially limited by the founders to Harvard students, but was expanded to other colleges in the Boston area, the Ivy League, and gradually most universities in the United States and Canada.",
+        "The history of natural language processing generally started in the 1950s, although work can be found from earlier periods. In 1950, Alan Turing published an article titled 'Computing Machinery and Intelligence' which proposed what is now called the Turing test as a criterion of intelligence",
+        "Geoffrey Everest Hinton is an English Canadian cognitive psychologist and computer scientist, most noted for his work on artificial neural networks. Since 2013 he divides his time working for Google and the University of Toronto. In 2017, he cofounded and became the Chief Scientific Advisor of the Vector Institute in Toronto.",
+        "When I told John that I wanted to move to Alaska, he warned me that I'd have trouble finding a Starbucks there.",
+        "Steven Paul Jobs was an American business magnate, industrial designer, investor, and media proprietor. He was the chairman, chief executive officer (CEO), and co-founder of Apple Inc., the chairman and majority shareholder of Pixar, a member of The Walt Disney Company's board of directors following its acquisition of Pixar, and the founder, chairman, and CEO of NeXT. Jobs is widely recognized as a pioneer of the personal computer revolution of the 1970s and 1980s, along with Apple co-founder Steve Wozniak. Jobs was born in San Francisco, California, and put up for adoption. He was raised in the San Francisco Bay Area. He attended Reed College in 1972 before dropping out that same year, and traveled through India in 1974 seeking enlightenment and studying Zen Buddhism.",
+        "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+        "Other than being the king of the north, John Snow is a an english physician and a leader in the development of anaesthesia and medical hygiene. He is considered for being the first one using data to cure cholera outbreak in 1834."
+    ],
+    "Zero-Shot Classification" : [
+        "In today’s world, staying updated with urgent information is crucial as events can unfold rapidly and require immediate attention.",  # Urgent
+        "Mobile technology has become indispensable, allowing us to access news, updates, and connect with others no matter where we are.",  # Mobile
+        "For those who love to travel, the convenience of mobile apps has transformed how we plan and experience trips, providing real-time updates on flights, accommodations, and local attractions.",  # Travel
+        "The entertainment industry continually offers new movies that captivate audiences with their storytelling and visuals, providing a wide range of genres to suit every taste.",  # Movie
+        "Music is an integral part of modern life, with streaming platforms making it easy to discover new artists and enjoy favorite tunes anytime, anywhere.",  # Music
+        "Sports enthusiasts follow games and matches closely, with live updates and detailed statistics available at their fingertips, enhancing the excitement of every game.",  # Sport
+        "Weather forecasts play a vital role in daily planning, offering accurate and timely information to help us prepare for various weather conditions and adjust our plans accordingly.",  # Weather
+        "Technology continues to evolve rapidly, driving innovation across all sectors and improving our everyday lives through smarter devices, advanced software, and enhanced connectivity."  # Technology
+    ],
+    "Sequence Classification": [
+        "This movie was absolutely fantastic! The storyline was gripping, the characters were well-developed, and the cinematography was stunning. I was on the edge of my seat the entire time.",
+        "A heartwarming and beautiful film. The performances were top-notch, and the direction was flawless. This is easily one of the best movies I've seen this year.",
+        "What a delightful surprise! The humor was spot on, and the plot was refreshingly original. The cast did an amazing job bringing the characters to life. Highly recommended!",
+        "This was one of the worst movies I’ve ever seen. The plot was predictable, the acting was wooden, and the pacing was painfully slow. I couldn’t wait for it to end.",
+        "A complete waste of time. The movie lacked any real substance or direction, and the dialogue was cringe-worthy. I wouldn’t recommend this to anyone.",
+        "I had high hopes for this film, but it turned out to be a huge disappointment. The story was disjointed, and the special effects were laughably bad. Don’t bother watching this one.",
+        "The movie was okay, but nothing special. It had a few good moments, but overall, it felt pretty average. Not something I would watch again, but it wasn’t terrible either.",
+        "An average film with a decent plot. The acting was passable, but it didn't leave much of an impression on me. It's a movie you might watch once and forget about.",
+        "This movie was neither good nor bad, just kind of there. It had some interesting ideas, but they weren’t executed very well. It’s a film you could take or leave."
+    ],
+    "Question Answering": {
+        """What does increased oxygen concentrations in the patient’s lungs displace?""": """Hyperbaric (high-pressure) medicine uses special oxygen chambers to increase the partial pressure of O 2 around the patient and, when needed, the medical staff. Carbon monoxide poisoning, gas gangrene, and decompression sickness (the ’bends’) are sometimes treated using these devices. Increased O 2 concentration in the lungs helps to displace carbon monoxide from the heme group of hemoglobin. Oxygen gas is poisonous to the anaerobic bacteria that cause gas gangrene, so increasing its partial pressure helps kill them. Decompression sickness occurs in divers who decompress too quickly after a dive, resulting in bubbles of inert gas, mostly nitrogen and helium, forming in their blood. Increasing the pressure of O 2 as soon as possible is part of the treatment.""",
+        """What category of game is Legend of Zelda: Twilight Princess?""": """The Legend of Zelda: Twilight Princess (Japanese: ゼルダの伝説 トワイライトプリンセス, Hepburn: Zeruda no Densetsu: Towairaito Purinsesu?) is an action-adventure game developed and published by Nintendo for the GameCube and Wii home video game consoles. It is the thirteenth installment in the The Legend of Zelda series. Originally planned for release on the GameCube in November 2005, Twilight Princess was delayed by Nintendo to allow its developers to refine the game, add more content, and port it to the Wii. The Wii version was released alongside the console in North America in November 2006, and in Japan, Europe, and Australia the following month. The GameCube version was released worldwide in December 2006.""",
+        """Who is founder of Alibaba Group?""": """Alibaba Group founder Jack Ma has made his first appearance since Chinese regulators cracked down on his business empire. His absence had fuelled speculation over his whereabouts amid increasing official scrutiny of his businesses. The billionaire met 100 rural teachers in China via a video meeting on Wednesday, according to local government media. Alibaba shares surged 5% on Hong Kong's stock exchange on the news.""",
+        """For what instrument did Frédéric write primarily for?""": """Frédéric François Chopin (/ˈʃoʊpæn/; French pronunciation: ​[fʁe.de.ʁik fʁɑ̃.swa ʃɔ.pɛ̃]; 22 February or 1 March 1810 – 17 October 1849), born Fryderyk Franciszek Chopin,[n 1] was a Polish and French (by citizenship and birth of father) composer and a virtuoso pianist of the Romantic era, who wrote primarily for the solo piano. He gained and has maintained renown worldwide as one of the leading musicians of his era, whose "poetic genius was based on a professional technique that was without equal in his generation." Chopin was born in what was then the Duchy of Warsaw, and grew up in Warsaw, which after 1815 became part of Congress Poland. A child prodigy, he completed his musical education and composed his earlier works in Warsaw before leaving Poland at the age of 20, less than a month before the outbreak of the November 1830 Uprising.""",
+        """The most populated city in the United States is which city?""": """New York—often called New York City or the City of New York to distinguish it from the State of New York, of which it is a part—is the most populous city in the United States and the center of the New York metropolitan area, the premier gateway for legal immigration to the United States and one of the most populous urban agglomerations in the world. A global power city, New York exerts a significant impact upon commerce, finance, media, art, fashion, research, technology, education, and entertainment, its fast pace defining the term New York minute. Home to the headquarters of the United Nations, New York is an important center for international diplomacy and has been described as the cultural and financial capital of the world."""
+    }
+}
+
+if task == 'Question Answering':
+    examples = list(examples_mapping[task].keys())
+    selected_text = st.selectbox('Select an Example:', examples)
+    st.subheader('Try it yourself!')
+    custom_input_question = st.text_input('Create a question')
+    custom_input_context = st.text_input("Create it's context")
+
+    custom_examples = {}
+
+    st.subheader('Selected Text')
+
+    if custom_input_question and custom_input_context:
+        QUESTION = custom_input_question
+        CONTEXT = custom_input_context
+    elif selected_text:
+        QUESTION = selected_text
+        CONTEXT = examples_mapping[task][selected_text]
+
+    st.markdown(f"**Question:** {QUESTION}")
+    st.markdown(f"**Context:** {CONTEXT}")
+        
+else:
+    examples = examples_mapping[task]
+    selected_text = st.selectbox("Select an example", examples)
+    custom_input = st.text_input("Try it with your own Sentence!")
+
+if task == 'Zero-Shot Classification':
+    zeroShotLables = ["urgent", "mobile", "travel", "movie", "music", "sport", "weather", "technology"]
+    lables = st_tags(
+        label='Select labels',
+        text='Press enter to add more',
+        value=zeroShotLables,
+        suggestions=[
+          "Positive", "Negative", "Neutral",
+          "Urgent", "Mobile", "Travel", "Movie", "Music", "Sport", "Weather", "Technology",
+          "Happiness", "Sadness", "Anger", "Fear", "Surprise", "Disgust",
+          "Informational", "Navigational", "Transactional", "Commercial Investigation",
+          "Politics", "Business", "Sports", "Entertainment", "Health", "Science",
+          "Product Quality", "Delivery Experience", "Customer Service", "Pricing", "Return Policy",
+          "Education", "Finance", "Lifestyle", "Fashion", "Food", "Art", "History",
+          "Culture", "Environment", "Real Estate", "Automotive", "Travel", "Fitness", "Career"],
+        maxtags = -1)
+
+try:
+  text_to_analyze = custom_input if custom_input else selected_text
+  st.subheader('Full example text')
+  HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+  st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+except:
+  text_to_analyze = selected_text
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+
+if task == 'Zero-Shot Classification':
+    pipeline = create_pipeline(model, task, zeroShotLables)
+else:
+    pipeline = create_pipeline(model, task)
+
+try:
+  output = fit_data(pipeline, text_to_analyze, task, QUESTION, CONTEXT)
+except:
+  output = fit_data(pipeline, text_to_analyze, task)
+
+# Display matched sentence
+st.subheader("Prediction:")
+
+if task == 'Token Classification':
+    abbreviation_mapping = {'R': 'PER', 'G': 'ORG', 'C': 'LOC', 'SC': 'MISC'}
+    results = {
+        'Document': output[0]['document'][0].result,
+        'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+        'NER Label': [abbreviation_mapping.get(n.metadata['entity'], 'UNKNOWN') for n in output[0]['ner_chunk']]
+    }
+    annotate(results)
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)
+
+elif task == 'Zero-Shot Classification':
+    st.markdown(f"Document Classified as: **{output[0]['class'][0].result}**")
+
+elif task == 'Sequence Classification':
+    st.markdown(f"Classified as : **{output[0]['class'][0].result}**")
+    
+elif task == "Question Answering":
+    output_text = "".join(output[0][0])
+    st.markdown(f"Answer: **{output_text}**")
+
+
+    

Demo.py

@@ -0,0 +1,248 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+from streamlit_tags import st_tags
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model, task, zeroShotLables=['']):
+    document_assembler = DocumentAssembler() \
+        .setInputCol('text') \
+        .setOutputCol('document')
+
+    sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\
+        .setInputCols(["document"])\
+        .setOutputCol("sentence")
+
+    tokenizer = Tokenizer() \
+        .setInputCols(['sentence']) \
+        .setOutputCol('token')
+
+    if task == "Token Classification":
+        TCclassifier = DeBertaForTokenClassification \
+            .pretrained(model, "en") \
+            .setInputCols(["sentence", "token"]) \
+            .setOutputCol("ner") \
+            .setCaseSensitive(True) \
+            .setMaxSentenceLength(512)
+        
+        ner_converter = NerConverter() \
+            .setInputCols(['sentence', 'token', 'ner']) \
+            .setOutputCol('ner_chunk')
+
+        TCpipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, TCclassifier, ner_converter])
+        return TCpipeline
+
+    elif task == "Zero-Shot Classification":
+        ZSCtokenizer = Tokenizer() \
+            .setInputCols(['document']) \
+            .setOutputCol('token')
+        
+        zeroShotClassifier = DeBertaForZeroShotClassification \
+            .pretrained('deberta_base_zero_shot_classifier_mnli_anli_v3', 'en') \
+            .setInputCols(['token', 'document']) \
+            .setOutputCol('class') \
+            .setCaseSensitive(False) \
+            .setMaxSentenceLength(512) \
+            .setCandidateLabels(zeroShotLables)
+
+        ZSCpipeline = Pipeline(stages=[document_assembler, ZSCtokenizer, zeroShotClassifier])
+        return ZSCpipeline
+
+    elif task == "Sequence Classification":
+        SCtokenizer = Tokenizer() \
+            .setInputCols(['document']) \
+            .setOutputCol('token')
+
+        sequence_classifier = DeBertaForSequenceClassification \
+            .pretrained("deberta_v3_base_sequence_classifier_imdb", "en") \
+            .setInputCols(["document", "token"]) \
+            .setOutputCol("class")
+        
+        SCpipeline = Pipeline(stages=[document_assembler, SCtokenizer, sequence_classifier])
+        return SCpipeline
+
+def fit_data(pipeline, data, task):
+    if task in ['Token Classification', 'Sequence Classification']:
+      empty_df = spark.createDataFrame([['']]).toDF('text')
+      pipeline_model = pipeline.fit(empty_df)
+      model = LightPipeline(pipeline_model)
+      result = model.fullAnnotate(data)
+      return result
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+tasks_models_descriptions = {
+    "Token Classification": {
+        "models": ["deberta_v3_small_token_classifier_conll03", "deberta_v3_large_token_classifier_conll03"],
+        "description": "The 'deberta_v3_small_token_classifier_conll03' model is adept at token classification tasks, including named entity recognition (NER). It identifies and categorizes tokens in text, such as names, dates, and locations, enhancing the extraction of meaningful information from unstructured data."
+    },
+    "Zero-Shot Classification": {
+        "models": ["deberta_base_zero_shot_classifier_mnli_anli_v3"],
+        "description": "The 'deberta_base_zero_shot_classifier_mnli_anli_v3' model provides flexible text classification without needing training data for specific categories. It is ideal for dynamic scenarios where text needs to be categorized into topics like urgent issues, technology, or sports without prior labeling."
+    },
+    "Sequence Classification": {
+        "models": ["deberta_v3_base_sequence_classifier_imdb"],
+        "description": "The 'deberta_v3_base_sequence_classifier_imdb' model is proficient in sequence classification tasks, such as sentiment analysis and document categorization. It effectively determines the sentiment of reviews, classifies text, and sorts documents based on their content and context."
+    }
+}
+
+# Sidebar content
+task = st.sidebar.selectbox("Choose the task", list(tasks_models_descriptions.keys()))
+model = st.sidebar.selectbox("Choose the pretrained model", tasks_models_descriptions[task]["models"], help="For more info about the models visit: https://sparknlp.org/models")
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/357691d18373d6e8f13b5b1015137a398fd0a45f/Spark_NLP_Udemy_MOOC/Open_Source/17.01.Transformers-based_Embeddings.ipynb#L103">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Page content
+title, sub_title = (f'DeBERTa for {task}', tasks_models_descriptions[task]["description"])
+st.markdown(f'<div class="main-title">{title}</div>', unsafe_allow_html=True)
+container = st.container(border=True)
+container.write(sub_title)
+
+# Load examples
+examples_mapping = {
+    "Token Classification": [
+        "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+        "The Mona Lisa is a 16th century oil painting created by Leonardo. It's held at the Louvre in Paris.",
+        "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously. “I can tell you very senior CEOs of major American car companies would shake my hand and turn away because I wasn’t worth talking to,” said Thrun, now the co-founder and CEO of online higher education startup Udacity, in an interview with Recode earlier this week.",
+        "Facebook is a social networking service launched as TheFacebook on February 4, 2004. It was founded by Mark Zuckerberg with his college roommates and fellow Harvard University students Eduardo Saverin, Andrew McCollum, Dustin Moskovitz and Chris Hughes. The website's membership was initially limited by the founders to Harvard students, but was expanded to other colleges in the Boston area, the Ivy League, and gradually most universities in the United States and Canada.",
+        "The history of natural language processing generally started in the 1950s, although work can be found from earlier periods. In 1950, Alan Turing published an article titled 'Computing Machinery and Intelligence' which proposed what is now called the Turing test as a criterion of intelligence",
+        "Geoffrey Everest Hinton is an English Canadian cognitive psychologist and computer scientist, most noted for his work on artificial neural networks. Since 2013 he divides his time working for Google and the University of Toronto. In 2017, he cofounded and became the Chief Scientific Advisor of the Vector Institute in Toronto.",
+        "When I told John that I wanted to move to Alaska, he warned me that I'd have trouble finding a Starbucks there.",
+        "Steven Paul Jobs was an American business magnate, industrial designer, investor, and media proprietor. He was the chairman, chief executive officer (CEO), and co-founder of Apple Inc., the chairman and majority shareholder of Pixar, a member of The Walt Disney Company's board of directors following its acquisition of Pixar, and the founder, chairman, and CEO of NeXT. Jobs is widely recognized as a pioneer of the personal computer revolution of the 1970s and 1980s, along with Apple co-founder Steve Wozniak. Jobs was born in San Francisco, California, and put up for adoption. He was raised in the San Francisco Bay Area. He attended Reed College in 1972 before dropping out that same year, and traveled through India in 1974 seeking enlightenment and studying Zen Buddhism.",
+        "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+        "Other than being the king of the north, John Snow is a an english physician and a leader in the development of anaesthesia and medical hygiene. He is considered for being the first one using data to cure cholera outbreak in 1834."
+    ],
+    "Zero-Shot Classification" : [
+        "In today’s world, staying updated with urgent information is crucial as events can unfold rapidly and require immediate attention.",  # Urgent
+        "Mobile technology has become indispensable, allowing us to access news, updates, and connect with others no matter where we are.",  # Mobile
+        "For those who love to travel, the convenience of mobile apps has transformed how we plan and experience trips, providing real-time updates on flights, accommodations, and local attractions.",  # Travel
+        "The entertainment industry continually offers new movies that captivate audiences with their storytelling and visuals, providing a wide range of genres to suit every taste.",  # Movie
+        "Music is an integral part of modern life, with streaming platforms making it easy to discover new artists and enjoy favorite tunes anytime, anywhere.",  # Music
+        "Sports enthusiasts follow games and matches closely, with live updates and detailed statistics available at their fingertips, enhancing the excitement of every game.",  # Sport
+        "Weather forecasts play a vital role in daily planning, offering accurate and timely information to help us prepare for various weather conditions and adjust our plans accordingly.",  # Weather
+        "Technology continues to evolve rapidly, driving innovation across all sectors and improving our everyday lives through smarter devices, advanced software, and enhanced connectivity."  # Technology
+    ],
+    "Sequence Classification": [
+        "This movie was absolutely fantastic! The storyline was gripping, the characters were well-developed, and the cinematography was stunning. I was on the edge of my seat the entire time.",
+        "A heartwarming and beautiful film. The performances were top-notch, and the direction was flawless. This is easily one of the best movies I've seen this year.",
+        "What a delightful surprise! The humor was spot on, and the plot was refreshingly original. The cast did an amazing job bringing the characters to life. Highly recommended!",
+        "This was one of the worst movies I’ve ever seen. The plot was predictable, the acting was wooden, and the pacing was painfully slow. I couldn’t wait for it to end.",
+        "A complete waste of time. The movie lacked any real substance or direction, and the dialogue was cringe-worthy. I wouldn’t recommend this to anyone.",
+        "I had high hopes for this film, but it turned out to be a huge disappointment. The story was disjointed, and the special effects were laughably bad. Don’t bother watching this one.",
+        "The movie was okay, but nothing special. It had a few good moments, but overall, it felt pretty average. Not something I would watch again, but it wasn’t terrible either.",
+        "An average film with a decent plot. The acting was passable, but it didn't leave much of an impression on me. It's a movie you might watch once and forget about.",
+        "This movie was neither good nor bad, just kind of there. It had some interesting ideas, but they weren’t executed very well. It’s a film you could take or leave."
+    ]
+}
+
+examples = examples_mapping[task]
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+if task == 'Zero-Shot Classification':
+    zeroShotLables = ["urgent", "mobile", "travel", "movie", "music", "sport", "weather", "technology"]
+    lables = st_tags(
+        label='Select labels',
+        text='Press enter to add more',
+        value=zeroShotLables,
+        suggestions=[
+          "Positive", "Negative", "Neutral",
+          "Urgent", "Mobile", "Travel", "Movie", "Music", "Sport", "Weather", "Technology",
+          "Happiness", "Sadness", "Anger", "Fear", "Surprise", "Disgust",
+          "Informational", "Navigational", "Transactional", "Commercial Investigation",
+          "Politics", "Business", "Sports", "Entertainment", "Health", "Science",
+          "Product Quality", "Delivery Experience", "Customer Service", "Pricing", "Return Policy",
+          "Education", "Finance", "Lifestyle", "Fashion", "Food", "Art", "History",
+          "Culture", "Environment", "Real Estate", "Automotive", "Travel", "Fitness", "Career"],
+        maxtags = -1)
+
+try:
+  text_to_analyze = custom_input if custom_input else selected_text
+  st.subheader('Full example text')
+  HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+  st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+except:
+  text_to_analyze = selected_text
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model, task)
+output = fit_data(pipeline, text_to_analyze, task)
+
+# Display matched sentence
+st.subheader("Prediction:")
+
+if task == 'Token Classification':
+    results = {
+        'Document': output[0]['document'][0].result,
+        'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+        'NER Label': [n.metadata['entity'] for n in output[0]['ner_chunk']]
+    }
+    annotate(results)
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)
+
+elif task == 'Zero-Shot Classification':
+    st.markdown(f"Document Classified as: **{output[0]['class'][0].result}**")
+
+elif task == 'Sequence Classification':
+    st.markdown(f"Classified as : **{output[0]['class'][0].result}**")
+    
+    

Dockerfile

@@ -0,0 +1,72 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean
+
+# Set up JAVA_HOME
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> /etc/profile \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set up PySpark to use Python 3.8 for both driver and workers
+ENV PYSPARK_PYTHON=/usr/bin/python3.8
+ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3.8
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,479 @@
+import streamlit as st
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+        .benchmark-table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 20px;
+        }
+        .benchmark-table th, .benchmark-table td {
+            border: 1px solid #ddd;
+            padding: 8px;
+            text-align: left;
+        }
+        .benchmark-table th {
+            background-color: #4A90E2;
+            color: white;
+        }
+        .benchmark-table td {
+            background-color: #f2f2f2;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Title
+st.markdown('<div class="main-title">Introduction to DeBERTa Annotators in Spark NLP</div>', unsafe_allow_html=True)
+
+# Subtitle
+st.markdown("""
+<div class="section">
+    <p>DeBERTa (Decoding-enhanced BERT with Disentangled Attention) is an advanced language model that builds upon BERT and RoBERTa, incorporating novel techniques such as disentangled attention and enhanced mask decoding. DeBERTa models excel in various NLP tasks, including text classification, token classification, masked language modeling, and question answering. Below, we provide an overview of the DeBERTa annotators for these tasks:</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Tabs for DeBERTa Annotators
+tab1, tab2, tab3, tab4 = st.tabs(["DeBERTa for Token Classification", "DeBERTa for Sequence Classification", "DeBERTa for Zero Shot Classification", "DeBERTa for Question Answering"])
+
+# Tab 1: DeBERTa for Token Classification
+with tab1:
+    st.markdown("""
+    <div class="section">
+        <h2>Token Classification with Spark NLP</h2>
+        <p>The <strong>Token Classification</strong> task is a core component of Natural Language Processing (NLP), focusing on classifying tokens (words or subwords) in a text into predefined categories. This task is fundamental for various applications, such as Named Entity Recognition (NER), Part-of-Speech (POS) tagging, and more.</p>
+        <p>Spark NLP offers a robust suite of tools for token classification, leveraging state-of-the-art models like BERT, RoBERTa, and DeBERTa. These models have been fine-tuned on diverse datasets and are readily available in Spark NLP to cater to a wide range of token classification tasks.</p>
+        <p>Token classification with Spark NLP enables:</p>
+        <ul>
+            <li><strong>Named Entity Recognition (NER):</strong> Recognizing and categorizing entities such as locations (LOC), organizations (ORG), persons (PER), and more.</li>
+            <li><strong>Information Extraction:</strong> Extracting structured data from unstructured text for deeper analysis and processing.</li>
+            <li><strong>Text Categorization:</strong> Enhancing document retrieval, classification, and organization based on identified entities.</li>
+        </ul>
+        <p>Using Spark NLP for token classification tasks offers several advantages:</p>
+        <ul>
+            <li><strong>Scalability:</strong> Spark NLP is designed to scale seamlessly with Apache Spark, making it suitable for processing large volumes of text data efficiently.</li>
+            <li><strong>Flexibility:</strong> A wide array of pre-trained models are available, allowing you to select the model that best fits your specific task, whether it’s for NER, POS tagging, or another classification task.</li>
+            <li><strong>Ease of Use:</strong> Spark NLP integrates smoothly into your existing Spark pipeline, allowing for quick and easy implementation.</li>
+            <li><strong>Customization:</strong> Models can be fine-tuned or adapted to new domains, giving you the flexibility to tailor the solution to your specific needs.</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # General Information about Using Token Classification Models
+    st.markdown('<div class="sub-title">How to Use Token Classification Models in Spark NLP</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>To perform token classification in Spark NLP, one powerful model you can use is DeBERTa, which stands for <strong>Decoding-enhanced BERT with Disentangled Attention</strong>. DeBERTa improves upon earlier models like BERT and RoBERTa, offering superior performance in tasks such as Named Entity Recognition (NER). Below is a template for setting up a token classification pipeline in Spark NLP using DeBERTa. This approach is flexible, allowing you to adjust the pipeline and parameters to meet your specific needs while leveraging DeBERTa's advanced capabilities.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    document_assembler = DocumentAssembler() \\
+        .setInputCol("text") \\
+        .setOutputCol("document")
+
+    tokenizer = Tokenizer() \\
+        .setInputCols(["document"]) \\
+        .setOutputCol("token")
+
+    # Example of loading a token classification model (e.g., BERT, RoBERTa, DeBERTa)
+    tokenClassifier = DeBertaForTokenClassification \\
+        .pretrained("deberta_v3_small_token_classifier_conll03", "en") \\
+        .setInputCols(["document", "token"]) \\
+        .setOutputCol("ner") \\
+        .setCaseSensitive(True) \\
+        .setMaxSentenceLength(512)
+
+    ner_converter = NerConverter() \\
+        .setInputCols(['document', 'token', 'ner']) \\
+        .setOutputCol('entities')
+
+    pipeline = Pipeline(stages=[
+        document_assembler,
+        tokenizer,
+        tokenClassifier,
+        ner_converter
+    ])
+
+    data = spark.createDataFrame([["Spark NLP is an exceptional library for NLP tasks."]]).toDF("text")
+    result = pipeline.fit(data).transform(data)
+    
+    result.selectExpr("explode(entities) as ner_chunk").select(
+        col("ner_chunk.result").alias("chunk"),
+        col("ner_chunk.metadata.entity").alias("ner_label")
+    ).show(truncate=False)
+    ''', language='python')
+
+    # Results Example
+    st.text("""
+    +--------------------------+---------+
+    |chunk                     |ner_label|
+    +--------------------------+---------+
+    |Spark NLP                 |ORG      |
+    +--------------------------+---------+
+    """)
+
+    # Model Info Section
+    st.markdown('<div class="sub-title">Choosing the Right Model</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>Spark NLP offers a variety of pre-trained models for token classification tasks, including BERT, RoBERTa, DeBERTa, and more. The choice of model can significantly impact the accuracy and performance of your task.</p>
+        <p>To explore and choose the most suitable model for your specific needs, visit the <a class="link" href="https://sparknlp.org/models" target="_blank">Spark NLP Models Hub</a>. Here, you can find detailed information about each model, including its size, compatibility, and the specific tasks it excels at.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://sparknlp.org/docs/en/transformers#debertafortokenclassification" target="_blank">Transformers in Spark NLP</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+# Tab 2: DeBERTa for Sequence Classification
+with tab2:
+    st.markdown("""
+    <div class="section">
+        <h2>Sequence Classification with Spark NLP</h2>
+        <p><strong>Sequence Classification</strong> is a critical task in Natural Language Processing (NLP) where entire sequences of text (such as sentences or paragraphs) are classified into predefined categories. This task is essential for applications like sentiment analysis, document classification, and more.</p>
+        <p>Spark NLP offers robust tools for sequence classification, utilizing advanced models such as BERT, RoBERTa, and DeBERTa. These models are pre-trained on diverse datasets and are readily available within Spark NLP, enabling you to address a wide range of sequence classification challenges.</p>
+        <p>Sequence classification with Spark NLP supports:</p>
+        <ul>
+            <li><strong>Sentiment Analysis:</strong> Determining the sentiment expressed in a sequence, such as positive, negative, or neutral.</li>
+            <li><strong>Document Classification:</strong> Categorizing documents into various classes based on their content.</li>
+            <li><strong>Intent Detection:</strong> Identifying the underlying intent behind a sequence, often used in chatbot and virtual assistant applications.</li>
+        </ul>
+        <p>Leveraging Spark NLP for sequence classification offers several advantages:</p>
+        <ul>
+            <li><strong>Scalability:</strong> Spark NLP scales effortlessly with Apache Spark, making it well-suited for processing large-scale text data.</li>
+            <li><strong>Flexibility:</strong> A broad selection of pre-trained models is available, allowing you to choose the most appropriate model for your specific task.</li>
+            <li><strong>Ease of Integration:</strong> Spark NLP integrates smoothly into existing Spark pipelines, facilitating quick and efficient implementation.</li>
+            <li><strong>Customizability:</strong> Models can be fine-tuned or adapted to different domains, providing tailored solutions for specific needs.</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # General Information about Using Sequence Classification Models
+    st.markdown('<div class="sub-title">How to Use Sequence Classification Models in Spark NLP</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>For sequence classification in Spark NLP, one powerful model you can use is DeBERTa, which stands for <strong>Decoding-enhanced BERT with Disentangled Attention</strong>. DeBERTa offers enhanced performance compared to earlier models like BERT and RoBERTa, especially for tasks such as sentiment analysis. Below is a template for setting up a sequence classification pipeline in Spark NLP using DeBERTa. This approach is adaptable, allowing you to adjust the pipeline and parameters to suit your specific requirements while utilizing DeBERTa's advanced features.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    document_assembler = DocumentAssembler() \\
+        .setInputCol("text") \\
+        .setOutputCol("document")
+
+    tokenizer = Tokenizer() \\
+        .setInputCols(['document']) \\
+        .setOutputCol('token') 
+
+    # Example of loading a sequence classification model using DeBERTa
+    sequenceClassifier = DeBertaForSequenceClassification \\
+        .pretrained("deberta_v3_base_sequence_classifier_imdb", "en") \\
+        .setInputCols(["document", "token"]) \\
+        .setOutputCol("class") \\
+        .setCaseSensitive(True) \\
+        .setMaxSentenceLength(512) 
+
+    pipeline = Pipeline(stages=[
+        document_assembler,
+        tokenizer,
+        sequenceClassifier
+    ])
+
+    example = spark.createDataFrame([['I really liked that movie!']]).toDF("text")
+    result = pipeline.fit(example).transform(example)
+    
+    result.select("text", "class.result").show(truncate=False)
+    ''', language='python')
+
+    # Results Example
+    st.text("""
+    +------------------------------+---------+
+    |text                          |class    |
+    +------------------------------+---------+
+    |I really liked that movie!    |positive |
+    +------------------------------+---------+
+    """)
+
+    # Model Info Section
+    st.markdown('<div class="sub-title">Choosing the Right Model</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>Spark NLP provides a diverse range of pre-trained models for sequence classification tasks, including BERT, RoBERTa, DeBERTa, and more. The model you choose can greatly impact the accuracy and performance of your task.</p>
+        <p>To explore and select the model that best fits your specific needs, visit the <a class="link" href="https://sparknlp.org/models" target="_blank">Spark NLP Models Hub</a>. This resource offers detailed information about each model, including its size, compatibility, and the tasks it excels at.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # References Section
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://sparknlp.org/docs/en/transformers#debertaforsequenceclassification" target="_blank">Spark NLP Annotators Documentation</a></li>
+            <li><a class="link" href="https://arxiv.org/abs/2006.03654" target="_blank">DeBERTa: Decoding-enhanced BERT with Disentangled Attention (Research Paper)</a></li>
+            <li><a class="link" href="https://www.microsoft.com/en-us/research/project/deberta/" target="_blank">Microsoft Research: DeBERTa</a></li>
+            <li><a class="link" href="https://github.com/microsoft/DeBERTa" target="_blank">DeBERTa GitHub Repository</a></li>
+            <li><a class="link" href="https://paperswithcode.com/task/sequence-classification" target="_blank">Sequence Classification: Research Papers with Code</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+# Tab 3: DeBERTa for Zero Shot Classification
+with tab3:
+    st.markdown("""
+    <div class="section">
+        <h2>Zero-Shot Classification with Spark NLP</h2>
+        <p><strong>Zero-Shot Classification</strong> is a technique in Natural Language Processing (NLP) that allows a model to classify text into categories that it has not been explicitly trained on. This approach is particularly useful when you have new, unseen classes or labels that were not part of the training data.</p>
+        <p>Spark NLP provides powerful tools for zero-shot classification, leveraging models like DeBERTa. These models are trained to handle a wide range of classification tasks without requiring retraining on specific categories. This enables flexibility and adaptability for various classification needs.</p>
+        <p>Zero-shot classification with Spark NLP facilitates:</p>
+        <ul>
+            <li><strong>Dynamic Categorization:</strong> Classify text into new categories without additional training.</li>
+            <li><strong>Adaptability:</strong> Easily adapt to evolving classification needs and emerging topics.</li>
+            <li><strong>Cost Efficiency:</strong> Reduce the need for extensive retraining and model updates for new classification tasks.</li>
+        </ul>
+        <p>Using Spark NLP for zero-shot classification offers several benefits:</p>
+        <ul>
+            <li><strong>Scalability:</strong> Spark NLP integrates with Apache Spark, making it capable of handling large-scale text data efficiently.</li>
+            <li><strong>Flexibility:</strong> The zero-shot classification models can be used for various tasks without the need for task-specific retraining.</li>
+            <li><strong>Ease of Implementation:</strong> Seamlessly integrate zero-shot classification into existing Spark pipelines for efficient processing.</li>
+            <li><strong>Customizable:</strong> Define custom candidate labels and adapt the model to different classification needs.</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # General Information about Using Zero-Shot Classification Models
+    st.markdown('<div class="sub-title">How to Use Zero-Shot Classification Models in Spark NLP</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>For zero-shot classification in Spark NLP, one powerful model you can use is DeBERTa, which stands for <strong>Decoding-enhanced BERT with Disentangled Attention</strong>. DeBERTa's zero-shot classification capabilities enable it to classify text into categories without additional training on those specific categories. Below is a template for setting up a zero-shot classification pipeline in Spark NLP using DeBERTa. This approach is flexible, allowing you to adjust the pipeline and parameters to fit your specific needs while leveraging DeBERTa's advanced features.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    document_assembler = DocumentAssembler() \\
+        .setInputCol('text') \\
+        .setOutputCol('document')
+
+    tokenizer = Tokenizer() \\
+        .setInputCols(['document']) \\
+        .setOutputCol('token')
+
+    # Example of loading a zero-shot classification model using DeBERTa
+    zeroShotClassifier = DeBertaForZeroShotClassification \\
+        .pretrained('deberta_base_zero_shot_classifier_mnli_anli_v3', 'en') \\
+        .setInputCols(['token', 'document']) \\
+        .setOutputCol('class') \\
+        .setCaseSensitive(True) \\
+        .setMaxSentenceLength(512) \\
+        .setCandidateLabels(["urgent", "mobile", "travel", "movie", "music", "sport", "weather", "technology"])
+
+    pipeline = Pipeline(stages=[
+        document_assembler,
+        tokenizer,
+        zeroShotClassifier
+    ])
+
+    example = spark.createDataFrame([['I have a problem with my iphone that needs to be resolved asap!!']]).toDF("text")
+    result = pipeline.fit(example).transform(example)
+    
+    result.select("text", "class.result").show(truncate=False)
+    ''', language='python')
+
+    # Results Example
+    st.text("""
+    +------------------------------------------------------------+-------------+
+    |text                                                       |class        |
+    +------------------------------------------------------------+-------------+
+    |I have a problem with my iphone that needs to be resolved asap!!|mobile     |
+    +------------------------------------------------------------+-------------+
+    """)
+
+    # Model Info Section
+    st.markdown('<div class="sub-title">Choosing the Right Model</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>Spark NLP offers a variety of pre-trained models for zero-shot classification, including BERT, RoBERTa, and DeBERTa. These models are capable of handling a wide range of classification tasks without requiring additional training on specific categories.</p>
+        <p>To explore and select the most suitable model for your needs, visit the <a class="link" href="https://sparknlp.org/models" target="_blank">Spark NLP Models Hub</a>. Here, you can find detailed information about each model, including its size, compatibility, and the specific tasks it excels at.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # References Section
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://sparknlp.org/docs/en/transformers#distilbertforzeroshotclassification" target="_blank">Spark NLP DistilBertForZeroShotClassification Annotators Documentation</a></li>
+            <li><a class="link" href="https://arxiv.org/abs/2006.11477" target="_blank">Enabling Zero-Shot Classification Using Pre-trained Transformers (Research Paper)</a></li>
+            <li><a class="link" href="https://paperswithcode.com/task/zero-shot-classification" target="_blank">Zero-Shot Classification: Research Papers with Code</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+# Tab 4: DeBERTa for Question Answering
+with tab4:
+    st.markdown("""
+    <div class="section">
+        <h2>Question Answering with Spark NLP</h2>
+        <p><strong>Question Answering (QA)</strong> is a fundamental NLP task that involves building models capable of understanding and responding to questions based on a given context. This task is essential for applications such as chatbots, virtual assistants, and information retrieval systems.</p>
+        <p>Spark NLP provides robust tools for question answering, leveraging advanced models like DeBERTa. These models are trained to accurately identify and extract answers from a provided context, enhancing the effectiveness of QA systems.</p>
+        <p>Question answering with Spark NLP enables:</p>
+        <ul>
+            <li><strong>Automated Information Retrieval:</strong> Extracting relevant information from text to answer user queries.</li>
+            <li><strong>Interactive Systems:</strong> Enhancing chatbots and virtual assistants to provide accurate responses to user questions.</li>
+            <li><strong>Knowledge Extraction:</strong> Improving the ability to understand and leverage contextual information for various applications.</li>
+        </ul>
+        <p>Using Spark NLP for question answering offers several advantages:</p>
+        <ul>
+            <li><strong>Scalability:</strong> Spark NLP integrates with Apache Spark, making it suitable for handling large-scale QA tasks efficiently.</li>
+            <li><strong>Flexibility:</strong> The pre-trained models can be easily adapted to various QA scenarios and domains.</li>
+            <li><strong>Ease of Integration:</strong> Seamlessly integrate QA models into existing pipelines for efficient question answering.</li>
+            <li><strong>Customization:</strong> Fine-tune or adapt models to specific contexts or industries to improve performance.</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # General Information about Using Question Answering Models
+    st.markdown('<div class="sub-title">How to Use Question Answering Models in Spark NLP</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>For question answering in Spark NLP, you can utilize DeBERTa, which stands for <strong>Decoding-enhanced BERT with Disentangled Attention</strong>. The DeBERTa model for question answering is designed to extract precise answers from a given context in response to user queries. Below is a template for setting up a question answering pipeline in Spark NLP using DeBERTa. This approach allows you to effectively manage and process question-answering tasks.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    documentAssembler = MultiDocumentAssembler() \\
+        .setInputCols(["question", "context"]) \\
+        .setOutputCols(["document_question", "document_context"])
+
+    spanClassifier = DebertaForQuestionAnswering \\
+        .pretrained("deberta_v3_xsmall_qa_squad2", "en") \\
+        .setInputCols(["document_question", "document_context"]) \\
+        .setOutputCol("answer") \\
+        .setCaseSensitive(True)
+
+    pipeline = Pipeline(stages=[
+        documentAssembler,
+        spanClassifier
+    ])
+
+    data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context")
+    result = pipeline.fit(data).transform(data)
+    
+    result.select("question", "context", "answer.result").show(truncate=False)
+    ''', language='python')
+
+    # Results Example
+    st.text("""
+    +------------------------------+--------------------------------------------+------------------+
+    |question                     |context                                    |answer            |
+    +------------------------------+--------------------------------------------+------------------+
+    |What is my name?             |My name is Clara and I live in Berkeley.  |Clara             |
+    +------------------------------+--------------------------------------------+------------------+
+    """)
+
+    # Model Info Section
+    st.markdown('<div class="sub-title">Choosing the Right Model</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>Spark NLP offers a range of pre-trained models for question answering tasks, including DeBERTa and other advanced transformers. Selecting the right model can significantly impact the quality of your QA system.</p>
+        <p>To explore and select the most appropriate model for your QA needs, visit the <a class="link" href="https://sparknlp.org/models" target="_blank">Spark NLP Models Hub</a>. Here, you can find detailed information about each model, including its capabilities and performance.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # References Section
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://sparknlp.org/docs/en/transformers#debertaforsequenceclassification" target="_blank">Transformers in Spark NLP</a></li>
+            <li><a class="link" href="https://arxiv.org/abs/2005.00979" target="_blank">SQuAD2.0: The Stanford Question Answering Dataset (Research Paper)</a></li>
+            <li><a class="link" href="https://paperswithcode.com/task/question-answering" target="_blank">Question Answering: Research Papers with Code</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+
+# Footer
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Quick Links</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/docs/en/quickstart" target="_blank">Getting Started</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/models" target="_blank">Pretrained Models</a></li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english" target="_blank">Example Notebooks</a></li>
+        <li><a class="link" href="https://sparknlp.org/docs/en/install" target="_blank">Installation Guide</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+st-annotated-text
+streamlit-tags
+pandas
+numpy
+spark-nlp
+pyspark