Upload 5 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,141 @@
+import streamlit as st
+import sparknlp
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .scroll {
+            overflow-x: auto;
+            border: 1px solid #e6e9ef;
+            border-radius: 0.25rem;
+            padding: 1rem;
+            margin-bottom: 2.5rem;
+            white-space: pre-wrap;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model, task):
+    documentAssembler = DocumentAssembler() \
+        .setInputCol("text") \
+        .setOutputCol("documents")
+
+    t5 = T5Transformer.pretrained(model) \
+        .setTask(task) \
+        .setInputCols(["documents"]) \
+        .setMaxOutputLength(200) \
+        .setOutputCol("transfers")
+
+    pipeline = Pipeline().setStages([documentAssembler, t5])
+    return pipeline
+
+def fit_data(pipeline, data):
+    df = spark.createDataFrame([[data]]).toDF("text")
+    result = pipeline.fit(df).transform(df)
+    return result.select('transfers.result').collect()
+
+# Sidebar setup
+model = st.sidebar.selectbox(
+    "Choose the Pretrained Model",
+    ['t5_informal_to_formal_styletransfer', 't5_formal_to_informal_styletransfer'],
+    help="Select the model you want to use for style transfer."
+)
+
+# Reference notebook link in sidebar
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(
+    """
+    <a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5_LINGUISTIC.ipynb">
+        <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+    </a>
+    """, 
+    unsafe_allow_html=True
+)
+
+examples = {
+    "t5_informal_to_formal_styletransfer": [
+        "Who gives a crap about that anyway? It's not like it matters!",
+        "Hiya, how ya doing? I haven't seen ya in forever!",
+        "btw - ur face looks really familiar, have we met before?",
+        "I looooooooooooooove going to the movies! It's my absolute favorite thing to do!",
+        "Hey, what's up? Wanna grab a bite to eat later?",
+        "Nah, I'm good. Don't feel like going out tonight.",
+        "Yo, that was totally awesome! Can't believe we pulled it off!",
+        "Check this out, it's totally epic! You've gotta see it!",
+        "I'm so stoked for the weekend, can't wait to just chill!",
+        "Dude, that party was lit! Had the best time ever!"
+    ],
+    "t5_formal_to_informal_styletransfer": [
+        "Please leave the room now, as your presence is no longer required.",
+        "Thank you very much, sir! Your kindness is greatly appreciated.",
+        "It's a pleasure to meet you, and I look forward to our collaboration.",
+        "I appreciate your assistance with this matter. It was very helpful.",
+        "She understood the complex instructions very quickly and efficiently.",
+        "He contracted a fever after returning from his overseas trip.",
+        "He investigated his accountant thoroughly before making any decisions.",
+        "Kindly refrain from making any noise during the presentation.",
+        "She expressed her gratitude for the opportunity to work on this project.",
+        "He was extremely punctual and arrived precisely at the scheduled time."
+    ]
+}
+
+task_descriptions = {
+    "t5_informal_to_formal_styletransfer": "transfer Casual to Formal:",
+    "t5_formal_to_informal_styletransfer": "transfer Formal to Casual:"
+}
+
+# Set up the page layout
+title = "T5 for Informal to Formal Style Transfer"
+sub_title = "Effortlessly Transform Sentences and Explore Different Writing Styles"
+
+st.markdown(f'<div class="main-title">{title}</div>', unsafe_allow_html=True)
+st.markdown(f'<div style="text-align: center; color: #666666;">{sub_title}</div>', unsafe_allow_html=True)
+
+# Text selection and analysis
+selected_text = st.selectbox("Select an example", examples[model])
+custom_input = st.text_input("Try it with your own sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.write('Text to analyze:')
+st.markdown(f'<div class="scroll">{text_to_analyze}</div>', unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model, task_descriptions[model])
+output = fit_data(pipeline, text_to_analyze)
+
+# Display transformed sentence
+st.write("Predicted Sentence:")
+output_text = "".join(output[0][0])
+st.markdown(f'<div class="scroll">{output_text.title()}</div>', unsafe_allow_html=True)

Dockerfile

@@ -0,0 +1,72 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean
+
+# Set up JAVA_HOME
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> /etc/profile \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set up PySpark to use Python 3.8 for both driver and workers
+ENV PYSPARK_PYTHON=/usr/bin/python3.8
+ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3.8
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,197 @@
+import streamlit as st
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Title
+st.markdown('<div class="main-title">Switch Between Informal and Formal Style</div>', unsafe_allow_html=True)
+
+# Introduction Section
+st.markdown("""
+<div class="section">
+    <p>Switching between informal and formal styles is a crucial skill in effective communication. Informal style is often used in casual conversations, while formal style is reserved for professional or official contexts. Understanding how to adapt your language to different audiences can greatly enhance the clarity and impact of your message.</p>
+    <p>On this page, we explore how to implement a pipeline that automatically switches between informal and formal styles using advanced NLP models. We utilize the T5 Transformer model, fine-tuned for style transfer, to seamlessly convert sentences between these two styles.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# T5 Transformer Overview
+st.markdown('<div class="sub-title">Understanding the T5 Transformer for Style Transfer</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>The T5 (Text-To-Text Transfer Transformer) model, developed by Google, is a versatile tool for a wide range of NLP tasks. For style transfer, T5 can be fine-tuned to convert text between different styles, such as informal to formal and vice versa.</p>
+    <p>By processing input sentences and applying the appropriate style transfer, T5 generates outputs that adjust the tone while preserving the original meaning. This is especially useful for applications in writing assistance, automated editing, and communication training.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Performance Section
+st.markdown('<div class="sub-title">Performance and Use Cases</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>The T5 model has demonstrated strong performance in text transformation tasks, including style transfer between informal and formal language. It consistently produces accurate and contextually appropriate results, making it a valuable tool for enhancing communication in various settings.</p>
+    <p>This capability is beneficial for writers, editors, and professionals who need to adapt text to different audiences or contexts. The T5 model’s ability to perform these transformations efficiently makes it a powerful asset for improving written communication.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Implementation Section
+st.markdown('<div class="sub-title">Implementing Informal-Formal Style Switching</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>The following example demonstrates how to implement a style transfer pipeline using Spark NLP to switch between informal and formal styles. The pipeline includes a document assembler and the T5 model for performing the transformations in both directions.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code('''
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Initialize Spark NLP
+spark = sparknlp.start()
+
+# Define the pipeline stages
+document_assembler = DocumentAssembler()\\
+    .setInputCol("text")\\
+    .setOutputCol("documents")
+
+# Informal to Formal transformation
+t5_informal_to_formal = T5Transformer()\\
+    .pretrained("t5_informal_to_formal_styletransfer")\\
+    .setTask("transfer Casual to Formal:")\\
+    .setInputCols(["documents"])\\
+    .setOutputCol("formal")
+
+# Formal to Informal transformation
+t5_formal_to_informal = T5Transformer()\\
+    .pretrained("t5_formal_to_informal_styletransfer")\\
+    .setTask("transfer Formal to Casual:")\\
+    .setInputCols(["documents"])\\
+    .setOutputCol("informal")
+
+pipeline_informal_to_formal = Pipeline().setStages([document_assembler, t5_informal_to_formal])
+pipeline_formal_to_informal = Pipeline().setStages([document_assembler, t5_formal_to_informal])
+
+# Input data examples
+data_informal = spark.createDataFrame([["Hey, what’s up? Wanna hang out later?"]]).toDF("text")
+data_formal = spark.createDataFrame([["I would like to inquire about your availability for a meeting."]]).toDF("text")
+
+# Apply the pipeline for informal to formal
+result_informal_to_formal = pipeline_informal_to_formal.fit(data_informal).transform(data_informal)
+result_informal_to_formal.select("formal.result").show(truncate=False)
+
+# Apply the pipeline for formal to informal
+result_formal_to_informal = pipeline_formal_to_informal.fit(data_formal).transform(data_formal)
+result_formal_to_informal.select("informal.result").show(truncate=False)
+''', language='python')
+
+# Example Output
+st.text("""
++---------------------------------------------------------------+
+|formal.result                                                |
++---------------------------------------------------------------+
+|[I would like to know if you are available to meet later.]   |
++---------------------------------------------------------------+
+
++---------------------------------------------------------------+
+|informal.result                                              |
++---------------------------------------------------------------+
+|[Hey, wanna hang out later?]                                |
++---------------------------------------------------------------+
+""")
+
+# Model Info Section
+st.markdown('<div class="sub-title">Choosing the Right T5 Model for Style Transfer</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>For switching between informal and formal styles, we use the models: "t5_informal_to_formal_styletransfer" for informal-to-formal conversion and "t5_formal_to_informal_styletransfer" for formal-to-informal conversion.</p>
+    <p>Explore other T5 models tailored for different style transfer tasks on the <a class="link" href="https://sparknlp.org/models?annotator=T5Transformer" target="_blank">Spark NLP Models Hub</a> to find the best fit for your specific needs.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References Section
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html" target="_blank">Google AI Blog</a>: Exploring Transfer Learning with T5</li>
+        <li><a class="link" href="https://sparknlp.org/models?annotator=T5Transformer" target="_blank">Spark NLP Model Hub</a>: Explore T5 models</li>
+        <li>Model used for Informal to Formal: <a class="link" href="https://sparknlp.org/2022/05/31/t5_informal_to_formal_styletransfer_en_3_0.html" target="_blank">t5_informal_to_formal_styletransfer</a></li>
+        <li>Model used for Formal to Informal: <a class="link" href="https://sparknlp.org/2022/05/31/t5_formal_to_informal_styletransfer_en_3_0.html" target="_blank">t5_formal_to_informal_styletransfer</a></li>
+        <li><a class="link" href="https://github.com/google-research/text-to-text-transfer-transformer" target="_blank">GitHub</a>: T5 Transformer repository</li>
+        <li><a class="link" href="https://arxiv.org/abs/1910.10683" target="_blank">T5 Paper</a>: Detailed insights from the developers</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Community & Support Section
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Quick Links Section
+st.markdown('<div class="sub-title">Quick Links</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/docs/en/quickstart" target="_blank">Getting Started</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/models" target="_blank">Pretrained Models</a></li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english" target="_blank">Example Notebooks</a></li>
+        <li><a class="link" href="https://sparknlp.org/docs/en/install" target="_blank">Installation Guide</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+st-annotated-text
+streamlit-tags
+pandas
+numpy
+spark-nlp
+pyspark