Delete src

src/app/v1/app.py

@@ -1,329 +0,0 @@
-import os
-import sys
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
-import pandas as pd
-
-
-from src.rag.pipeline import RAGPipeline
-import streamlit as st
-from src.utils.data import (
-    build_filter,
-    get_filter_values,
-    get_meta,
-    load_json,
-    load_css,
-)
-from src.utils.writer import typewriter
-
-st.set_page_config(layout="wide")
-
-
-
-EMBEDDING_MODEL = "sentence-transformers/distiluse-base-multilingual-cased-v1"
-PROMPT_TEMPLATE = os.path.join("src", "rag", "prompt_template.yaml")
-
-
-@st.cache_data
-def load_css_style(path: str) -> None:
-    load_css(path)
-
-
-@st.cache_data
-def get_meta_data() -> pd.DataFrame:
-    return pd.read_csv(
-        os.path.join("database", "meta_data.csv"), dtype=({"retriever_id": str})
-    )
-
-
-@st.cache_data
-def get_authors_taxonomy() -> dict[str, list[str]]:
-    return load_json(os.path.join("data", "authors_filter.json"))
-
-
-@st.cache_data
-def get_draft_cat_taxonomy() -> dict[str, list[str]]:
-    return load_json(os.path.join("data", "draftcat_taxonomy_filter.json"))
-
-
-@st.cache_data
-def get_example_prompts() -> list[str]:
-    return [
-        example["question"]
-        for example in load_json(os.path.join("data", "example_prompts.json"))
-    ]
-
-
-@st.cache_resource
-def load_pipeline() -> RAGPipeline:
-    return RAGPipeline(
-        embedding_model=EMBEDDING_MODEL,
-        prompt_template=PROMPT_TEMPLATE,
-    )
-
-
-@st.cache_data
-def load_app_init() -> None:
-    # Define the title of the app
-    st.title("INC Plastic Treaty - Q&A")
-
-    # add warning emoji and style
-    st.markdown(
-        """
-        <p class="remark"> ⚠️ Remark:
-        The app is a beta version that serves as a basis for further development. We are aware that the performance is not yet sufficient and that the data basis is not yet complete. We are grateful for any feedback that contributes to the further development and improvement of the app!
-    """,
-        unsafe_allow_html=True,
-    )
-
-    # add explanation to the app
-    st.markdown(
-        """
-        <p class="description">
-        The app aims to facilitate the search for information and documents related to the UN Plastics Treaty Negotiations. The database includes all relevant documents that are available <a href=https://www.unep.org/inc-plastic-pollution target="_blank">here</a>. Users can query the data through a chatbot. Please note that, due to technical constraints, only a maximum of 10 documents can be used to generate the answer. A comprehensive response can therefore not be guaranteed. However, all relevant documents can be accessed via a link using the filter functions.
-        Filter functions are available to narrow down the data by country/author, zero draft categories and negotiation rounds. Pre-selecting relevant data enhances the accuracy of generated answers. Additionally, all documents selected via the filter function can be accessed via a link.
-        """,
-        unsafe_allow_html=True,
-    )
-
-
-load_css_style("style/style.css")
-
-
-# Load the data
-metadata = get_meta_data()
-authors_taxonomy = get_authors_taxonomy()
-draft_cat_taxonomy = get_draft_cat_taxonomy()
-example_prompts = get_example_prompts()
-
-# Load pipeline
-pipeline = load_pipeline()
-
-# Load app init
-load_app_init()
-
-
-filter_col = st.columns(1)
-# Filter column
-with filter_col[0]:
-    st.markdown("## Select Filters")
-    author_col, round_col, draft_cat_col = st.columns([1, 1, 1])
-
-    with author_col:
-        st.markdown("### Authors")
-        selected_author_parent = st.multiselect(
-            "Entity Parent", list(authors_taxonomy.keys())
-        )
-
-        available_child_items = []
-        for category in selected_author_parent:
-            available_child_items.extend(authors_taxonomy[category])
-
-        selected_authors = st.multiselect("Entity", available_child_items)
-
-    with round_col:
-        st.markdown("### Round")
-        negotiation_rounds = get_filter_values(metadata, "round")
-        selected_rounds = st.multiselect("Round", negotiation_rounds)
-
-    with draft_cat_col:
-        st.markdown("### Draft Categories")
-        selected_draft_cats_parent = st.multiselect(
-            "Draft Categories Parent", list(draft_cat_taxonomy.keys())
-        )
-        available_draft_cats_child_items = []
-        for category in selected_draft_cats_parent:
-            available_draft_cats_child_items.extend(draft_cat_taxonomy[category])
-
-        selected_draft_cats = st.multiselect(
-            "Draft Categories", available_draft_cats_child_items
-        )
-
-
-prompt_col, output_col = st.columns([1, 1.5])
-# make the buttons text smaller
-
-
-# GPT column
-with prompt_col:
-    st.markdown("## Filter documents")
-    st.markdown(
-        """
-        * The filter function allows you to see all documents that match the selected filters.
-        * Additionally, all documents selected via the filter function can be accessed via a link.
-        * Alternatively, you can ask a question to the model. The model will then provide you with an answer based on the filtered documents.
-        """
-    )
-    trigger_filter = st.session_state.setdefault("trigger", False)
-    if st.button("Filter documents"):
-        filter_selection_transformed = build_filter(
-            meta_data=metadata,
-            authors_filter=selected_authors,
-            draft_cats_filter=selected_draft_cats,
-            round_filter=selected_rounds,
-        )
-        documents = pipeline.document_store.get_all_documents(
-            filters=filter_selection_transformed
-        )
-        trigger_filter = True
-
-    st.markdown("## Ask a question")
-    if "prompt" not in st.session_state:
-        prompt = st.text_area("")
-    if (
-        "prompt" in st.session_state
-        and st.session_state.prompt in example_prompts  # noqa: E501
-    ):  # noqa: E501
-        prompt = st.text_area(
-            "Enter a question", value=st.session_state.prompt
-        )  # noqa: E501
-    if (
-        "prompt" in st.session_state
-        and st.session_state.prompt not in example_prompts  # noqa: E501
-    ):  # noqa: E501
-        del st.session_state["prompt"]
-        prompt = st.text_area("Enter a question")
-
-    trigger_ask = st.session_state.setdefault("trigger", False)
-    if st.button("Ask"):
-        with st.status("Filtering documents...", expanded=False) as status:
-            if filter_selection_transformed == {}:
-                st.warning(
-                    "No filters selected. We highly recommend to use filters otherwise the answer might not be accurate. In addition you might experience performance issues since the model has to analyze all available documents."
-                )
-            filter_selection_transformed = build_filter(
-                meta_data=metadata,
-                authors_filter=selected_authors,
-                draft_cats_filter=selected_draft_cats,
-                round_filter=selected_rounds,
-            )
-
-            documents = pipeline.document_store.get_all_documents(
-                filters=filter_selection_transformed
-            )
-            status.update(
-                label="Filtering documents completed!", state="complete", expanded=False
-            )
-        with st.status("Answering question...", expanded=True) as status:
-            result = pipeline(prompt=prompt, filters=filter_selection_transformed)
-            trigger_ask = True
-            status.update(
-                label="Answering question completed!", state="complete", expanded=False
-            )
-
-    st.markdown("### Examples")
-    st.markdown(
-        """
-        * These are example prompts that can be used to ask questions to the model
-        * Click on a prompt to use it as a question. You can also type your own question in the text area above.
-        * For questions like "How do country a, b and c [...]" please make sure to select the countries in the filter section. Otherwise the answer will not be accurate. In general we highly recommend to use the filter functions to narrow down the data.
-        """
-    )
-
-    for i, prompt in enumerate(example_prompts):
-        # with col[i % 4]:
-        if st.button(prompt):
-            if "key" not in st.session_state:
-                st.session_state["prompt"] = prompt
-    # Define the button
-
-
-if trigger_ask:
-    with output_col:
-        meta_data = get_meta(result=result)
-        answer = result["answers"][0].answer
-
-        meta_data_cleaned = []
-        seen_retriever_ids = set()
-
-        for data in meta_data:
-            retriever_id = data["retriever_id"]
-            content = data["content"]
-            if retriever_id not in seen_retriever_ids:
-                meta_data_cleaned.append(
-                    {
-                        "retriever_id": retriever_id,
-                        "href": data["href"],
-                        "content": [content],
-                    }
-                )
-                seen_retriever_ids.add(retriever_id)
-            else:
-                for i, item in enumerate(meta_data_cleaned):
-                    if item["retriever_id"] == retriever_id:
-                        meta_data_cleaned[i]["content"].append(content)
-
-        references = ["\n"]
-        for data in meta_data_cleaned:
-            retriever_id = data["retriever_id"]
-            href = data["href"]
-            references.append(f"-[{retriever_id}]: {href} \n")
-        st.write("#### 📌 Answer")
-        typewriter(
-            text=answer,
-            references=references,
-            speed=100,
-        )
-
-        with st.expander("Show more information to the documents"):
-            for data in meta_data_cleaned:
-                markdown_text = f"- Document: {data['retriever_id']}\n"
-                markdown_text += "    - Text passages\n"
-                for content in data["content"]:
-                    content = content.replace("[", "").replace("]", "").replace("'", "")
-                    content = " ".join(content.split())
-                    markdown_text += f"        - {content}\n"
-                st.write(markdown_text)
-
-    col4 = st.columns(1)
-    with col4[0]:
-        references = []
-        for document in documents:
-            authors = document.meta["author"]
-            authors = authors.replace("'", "").replace("[", "").replace("]", "")
-            href = document.meta["href"]
-            source = f"- {authors}: {href}"
-            references.append(source)
-            references = list(set(references))
-            references = sorted(references)
-        st.markdown("### Overview of all filtered documents")
-        st.markdown(
-            f"<p class='description'> The answer above results from the most similar text passages (top 7) from the documents that you can find under 'References' in the answer block. Below you will find an overview of all documents that match the filters you have selected. Please note that the above answer is based specifically on the highlighted references above and does not include the findings from all the filtered documents shown below. \n For your current filtering, {len(references)} documents were found. </p>",
-            unsafe_allow_html=True,
-        )
-        for reference in references:
-            st.write(reference)
-    trigger = 0
-
-
-if trigger_filter:
-    with output_col:
-        references = []
-        for document in documents:
-            authors = document.meta["author"]
-            authors = authors.replace("'", "").replace("[", "").replace("]", "")
-            href = document.meta["href"]
-            round_ = document.meta["round"]
-            draft_labs = document.meta["draft_labs"]
-            references.append(
-                {
-                    "author": authors,
-                    "href": href,
-                    "draft_labs": draft_labs,
-                    "round": round_,
-                }
-            )
-        references = pd.DataFrame(references)
-        references = references.drop_duplicates()
-        st.markdown("### Overview of all filtered documents")
-        # show
-        # make columns author and draft_labs bigger and make href width smaller and round width smaller
-        st.dataframe(
-            references,
-            hide_index=True,
-            column_config={
-                "author": st.column_config.ListColumn("Authors"),
-                "href": st.column_config.LinkColumn("Link to Document"),
-                "draft_labs": st.column_config.ListColumn("Draft Categories"),
-                "round": st.column_config.NumberColumn("Round"),
-            },
-        )

src/app/v2/app.py

@@ -1,385 +0,0 @@
-import os
-import sys
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
-import pandas as pd
-import streamlit as st
-import time
-
-
-from src.rag.pipeline import RAGPipeline
-from src.utils.data_v2 import (
-    build_filter,
-    get_meta,
-    load_json,
-    load_css,
-)
-from src.utils.writer import typewriter
-
-
-st.set_page_config(layout="wide")
-
-EMBEDDING_MODEL = "sentence-transformers/distiluse-base-multilingual-cased-v1"
-PROMPT_TEMPLATE = os.path.join("src", "rag", "prompt_template.yaml")
-
-
-@st.cache_data
-def load_css_style(path: str) -> None:
-    load_css(path)
-
-
-@st.cache_data
-def get_meta_data() -> pd.DataFrame:
-    return pd.read_csv(
-        os.path.join("database", "meta_data.csv"), dtype=({"retriever_id": str})
-    )
-
-
-@st.cache_data
-def get_df() -> pd.DataFrame:
-    return pd.read_csv(
-        os.path.join("data", "inc_df.csv"), dtype=({"retriever_id": str})
-    )[["retriever_id", "draft_labs", "author", "href", "round"]]
-
-
-@st.cache_data
-def get_authors_taxonomy() -> list[str]:
-    taxonomy = load_json(os.path.join("data", "authors_taxonomy.json"))
-    countries = []
-    members = taxonomy["Members"]
-    for key, value in members.items():
-        if key == "Countries" or key == "International and Regional State Associations":
-            countries.extend(value)
-    return countries
-
-
-@st.cache_data
-def get_draft_cat_taxonomy() -> dict[str, list[str]]:
-    taxonomy = load_json(os.path.join("data", "draftcat_taxonomy_filter.json"))
-    draft_labels = []
-    for _, subpart in taxonomy.items():
-        for label in subpart:
-            draft_labels.append(label)
-    return draft_labels
-
-
-@st.cache_data
-def get_example_prompts() -> list[str]:
-    return [
-        example["question"]
-        for example in load_json(os.path.join("data", "example_prompts.json"))
-    ]
-
-
-@st.cache_data
-def set_trigger_state_values() -> tuple[bool, bool]:
-    trigger_filter = st.session_state.setdefault("trigger", False)
-    trigger_ask = st.session_state.setdefault("trigger", False)
-    return trigger_filter, trigger_ask
-
-
-@st.cache_resource
-def load_pipeline() -> RAGPipeline:
-    return RAGPipeline(
-        embedding_model=EMBEDDING_MODEL,
-        prompt_template=PROMPT_TEMPLATE,
-    )
-
-
-@st.cache_data
-def load_app_init() -> None:
-    # Define the title of the app
-    st.title("INC Plastic Pollution Country Profile Analysis")
-
-    # add warning emoji and style
-
-    st.markdown(
-        """
-    <div class="remark">
-        <div class="remark-content">
-            <p class="remark-text" style="font-size: 20px;"> ⚠️ The app is a beta version that serves as a basis for further development. We are aware that the performance is not yet sufficient and that the data basis is not yet complete. We are grateful for any feedback that contributes to the further development and improvement of the app!</p>
-        </div>
-    </div>
-    """,
-        unsafe_allow_html=True,
-    )
-
-    st.markdown(
-        """
-        <a href="mailto:[email protected]" class="feedback-link">Send feedback</a>
-        """,
-        unsafe_allow_html=True,
-    )
-
-    # add explanation to the app
-    st.markdown(
-        """
-        <p class="description">
-        The app is tailored to enhance the efficiency of finding and accessing information on the UN Plastics Treaty Negotiations. It hosts a comprehensive database of relevant documents submitted by the members available <a href="https://www.unep.org/inc-plastic-pollution"> here</a>, which users can explore through an intuitive chatbot interface as well as simple filtering options.
-        The app excels in querying specific information about countries and their positions in  the negotiations, providing targeted and precise answers. However, it can process only up to 8 relevant documents at a time, which may limit responses to more complex inquiries. Filter options by authors and sections of the negotiation draft ensure the accuracy of the answers. Each document found via these filters is also directly accessible via a link, ensuring complete and easy access to the desired information.
-        </p>
-        """,
-        unsafe_allow_html=True,
-    )
-
-
-load_css_style("style/style.css")
-
-
-# Load the data
-df = get_df()
-df_transformed = get_meta_data()
-countries = get_authors_taxonomy()
-draft_labels = get_draft_cat_taxonomy()
-example_prompts = get_example_prompts()
-trigger_filter, trigger_ask = set_trigger_state_values()
-
-# Load pipeline
-pipeline = load_pipeline()
-
-# Load app init
-load_app_init()
-
-
-application_col = st.columns(1)
-
-
-with application_col[0]:
-    st.markdown("""<p class="header"> 1️⃣ Select countries""", unsafe_allow_html=True)
-    st.markdown(
-        """
-        <p class="description">
-        Please select the countries of interest. Your selection will refine the database to include documents submitted by these countries or recognized groupings such as Small Developing States, the African States Group, etc. </p>
-        """,
-        unsafe_allow_html=True,
-    )
-    selected_authors = st.multiselect(
-        label="country",
-        options=countries,
-        label_visibility="collapsed",
-        placeholder="Select country/countries",
-    )
-
-    st.write("\n")
-    st.write("\n")
-
-    st.markdown(
-        """<p class="header">  2️⃣ Select parts of the negotiation draft""",
-        unsafe_allow_html=True,
-    )
-    st.markdown(
-        """
-        <p class="description">
-        Please select the parts of the negotiation draft of interest. The negotiation draft can be accessed <a href="https://www.unep.org/inc-plastic-pollution/session-4/documents"> here</a>. </p>
-        """,
-        unsafe_allow_html=True,
-    )
-    selected_draft_cats = st.multiselect(
-        label="Subpart",
-        options=draft_labels,
-        label_visibility="collapsed",
-        placeholder="Select draft category/draft categories",
-    )
-
-    st.write("\n")
-    st.write("\n")
-
-    st.markdown(
-        """<p class="header"> 3️⃣ Ask a question or show documents based on selected filters""",
-        unsafe_allow_html=True,
-    )
-
-    asking, filtering = st.tabs(["Ask a question", "Filter documents"])
-
-    with filtering:
-        application_col_filter, output_col_filter = st.columns([1, 1.5])
-        # make the buttons text smaller
-        with application_col_filter:
-            st.markdown(
-                """
-                <p class="description">
-                This filter function allows you to see all documents that match the selected filters. The documents can be accessed via a link. \n
-                """,
-                unsafe_allow_html=True,
-            )
-            if st.button("Filter documents"):
-                filters, status = build_filter(
-                    meta_data=df_transformed,
-                    authors_filter=selected_authors,
-                    draft_cats_filter=selected_draft_cats,
-                )
-                if status == "no filters selected":
-                    st.info("No filters selected. All documents will be shown.")
-                    df_filtered = df[
-                        ["author", "href", "draft_labs", "round"]
-                    ].sort_values(by="author")
-                    trigger_filter = True
-                if status == "no results found":
-                    st.info(
-                        "No documents found for the combination of filters you've chosen. All countries are represented at least once in the data. Remove the draft categories to see all documents for the countries selected or try other draft categories."
-                    )
-                if status == "success":
-                    df_filtered = df[df["retriever_id"].isin(filters["retriever_id"])][
-                        ["author", "href", "draft_labs", "round"]
-                    ].sort_values(by="author")
-                    trigger_filter = True
-
-    with asking:
-        application_col_ask, output_col_ask = st.columns([1, 1.5])
-        with application_col_ask:
-            st.markdown(
-                """
-                <p class="description"> Ask a question, noting that the database has been restricted by filters and that your question should pertain to the selected data. \n
-                """,
-                unsafe_allow_html=True,
-            )
-            if "prompt" not in st.session_state:
-                prompt = st.text_area("Enter a question")
-            if (
-                "prompt" in st.session_state
-                and st.session_state.prompt in example_prompts  # noqa: E501
-            ):  # noqa: E501
-                prompt = st.text_area(
-                    "Enter a question", value=st.session_state.prompt
-                )  # noqa: E501
-            if (
-                "prompt" in st.session_state
-                and st.session_state.prompt not in example_prompts  # noqa: E501
-            ):  # noqa: E501
-                del st.session_state["prompt"]
-                prompt = st.text_area("Enter a question")
-
-            trigger_ask = st.session_state.setdefault("trigger", False)
-
-            if st.button("Ask"):
-                if prompt == "":
-                    st.error(
-                        "Please enter a question. Reloading the app in few seconds"
-                    )
-                    time.sleep(3)
-                    st.rerun()
-                with st.spinner("Filtering data...") as status:
-                    filter_selection_transformed, status = build_filter(
-                        meta_data=df_transformed,
-                        authors_filter=selected_authors,
-                        draft_cats_filter=selected_draft_cats,
-                    )
-
-                    if status == "no filters selected":
-                        st.info(
-                            "No filters selcted.This will increase the prcessing time significantly. Please select at least one filter."
-                        )
-                        # st.error(
-                        #     "Selecting a filter is mandatory. We especially recommend to select countries you are interested in. Selecting at least one filter is mandatory, because otherwise the model would have to analyze all available documents which results in inaccurate answers and long processing times. Please select at least one filter."
-                        # )
-                        # st.stop()
-
-                    documents = pipeline.document_store.get_all_documents(
-                        filters=filter_selection_transformed
-                    )
-
-                    st.success("Filtering data completed.")
-                with st.spinner("Answering question...") as status:
-                    if filter_selection_transformed == {}:
-                        st.warning(
-                            "The combination of filters you've chosen does not match any documents. Giving answer based on all documents. Please note that the answer might not be accurate. We highly recommend to use a combination of filters that match the data. All countries are represented at least once in the data. Thus, for example, you could remove the draft categories to match the documents. Or you could check with the Filter documents function which documents are available for the selected countries by removing the draft categories and filter the documents."
-                        )
-
-                    result = pipeline.run(
-                        prompt=prompt, filters=filter_selection_transformed
-                    )
-                    trigger_ask = True
-                    st.success("Answering question completed.")
-
-            st.markdown("### Examples")
-            for i, prompt in enumerate(example_prompts):
-                # with col[i % 4]:
-                if st.button(prompt):
-                    if "key" not in st.session_state:
-                        st.session_state["prompt"] = prompt
-            st.markdown(
-                """
-                <ul class="description" style="font-size: 20px;">
-                    <li style="font-size: 17px;">These are example prompts that can be used to ask questions to the model</li>
-                    <li style="font-size: 17px;">Click on a prompt to use it as a question. You can also type your own question in the text area above.</li>
-                    <li style="font-size: 17px;">For questions like "How do country a, b and c [...]" please make sure to select the countries in the filter section. Otherwise the answer will not be accurate. In general we highly recommend to use the filter functions to narrow down the data.</li>
-                </ul>
-                """,
-                unsafe_allow_html=True,
-            )
-
-            # for i, prompt in enumerate(example_prompts):
-            #     # with col[i % 4]:
-            #     if st.button(prompt):
-            #         if "key" not in st.session_state:
-            #             st.session_state["prompt"] = prompt
-            # Define the button
-
-    if trigger_ask:
-        with output_col_ask:
-            if result is None:
-                st.error(
-                    "Open AI rate limit exceeded. Please try again in a few seconds."
-                )
-                st.stop()
-            meta_data = get_meta(result=result)
-            answer = result["answers"][0].answer
-
-            meta_data_cleaned = []
-            seen_retriever_ids = set()
-
-            for data in meta_data:
-                retriever_id = data["retriever_id"]
-                content = data["content"]
-                if retriever_id not in seen_retriever_ids:
-                    meta_data_cleaned.append(
-                        {
-                            "retriever_id": retriever_id,
-                            "href": data["href"],
-                            "content": [content],
-                        }
-                    )
-                    seen_retriever_ids.add(retriever_id)
-                else:
-                    for i, item in enumerate(meta_data_cleaned):
-                        if item["retriever_id"] == retriever_id:
-                            meta_data_cleaned[i]["content"].append(content)
-
-            references = ["\n"]
-            for data in meta_data_cleaned:
-                retriever_id = data["retriever_id"]
-                href = data["href"]
-                references.append(f"-[{retriever_id}]: {href} \n")
-            st.write("#### 📌 Answer")
-            typewriter(
-                text=answer,
-                references=references,
-                speed=100,
-            )
-
-            with st.expander("Show more information to the documents"):
-                for data in meta_data_cleaned:
-                    markdown_text = f"- Document: {data['retriever_id']}\n"
-                    markdown_text += "    - Text passages\n"
-                    for content in data["content"]:
-                        content = (
-                            content.replace("[", "").replace("]", "").replace("'", "")
-                        )
-                        content = " ".join(content.split())
-                        markdown_text += f"        - {content}\n"
-                    st.write(markdown_text)
-
-        trigger = 0
-
-    if trigger_filter:
-        with output_col_filter:
-            st.markdown("### Overview of all filtered documents")
-            st.dataframe(
-                df_filtered,
-                hide_index=True,
-                column_config={
-                    "author": st.column_config.ListColumn("Authors"),
-                    "href": st.column_config.LinkColumn("Link to Document"),
-                    "draft_labs": st.column_config.ListColumn("Draft Categories"),
-                    "round": st.column_config.NumberColumn("Round"),
-                },
-            )

src/data_processing/document_store_data.py

@@ -1,93 +0,0 @@
-import pandas as pd
-import ast
-import json
-
-DATASET = "data/inc_df_v6_small_4.csv"
-DATASET_PROCESSED = "data/inc_df.csv"
-MEMBERS = "data/authors_filter.json"
-
-
-def main():
-    print(f"Length of dataset: {len(pd.read_csv(DATASET))}")
-    df = pd.read_csv(DATASET)
-    df["retriever_id"] = df.index
-    columns = [
-        "retriever_id",
-        "description",
-        "href",
-        "draft_labs_list",
-        "authors_list",
-        "draft_allcats",
-        "doc_subtype",
-        "doc_type",
-        "text",
-        "round",
-    ]
-
-    df = df[columns]
-
-    df.rename(
-        mapper={
-            "draft_labs_list": "draft_labs",
-            "draft_allcats": "draft_cats",
-            "authors_list": "author",
-        },
-        axis=1,
-        inplace=True,
-    )
-
-    ###Subselect for countries and country groups
-    with open(MEMBERS, "r") as f:
-        authors = json.load(f)
-    special_character_words_mapper = {
-        "Côte D'Ivoire": "Cote DIvoire",
-        "Ligue Camerounaise Des Droits De L'Homme": "Ligue Camerounaise Des Droits De LHomme",
-        "Association Pour L'Integration Et La Developpement Durable Au Burundi": "Association Pour LIntegration Et La Developpement Durable Au Burundi",
-    }
-    members = [
-        authors[key]
-        for key in [
-            "Members - Countries",
-            "Members - International and Regional State Associations",
-        ]
-    ]
-    members = [item for sublist in members for item in sublist]
-    members = [special_character_words_mapper.get(member, member) for member in members]
-
-    nonmembers = [
-        authors[key]
-        for key in [
-            "Intergovernmental Negotiation Committee",
-            "Observers and Other Participants",
-        ]
-    ]
-    nonmembers = [item for sublist in nonmembers for item in sublist]
-
-    df["author"][df["author"] == "['Côte D'Ivoire']"] = "['Cote DIvoire']"
-    df["author"][
-        df["author"] == "['Ligue Camerounaise Des Droits De L'Homme']"
-    ] = "['Ligue Camerounaise Des Droits De LHomme']"
-    df["author"][
-        df["author"]
-        == "['Association Pour L'Integration Et La Developpement Durable Au Burundi']"
-    ] = "['Association Pour LIntegration Et La Developpement Durable Au Burundi']"
-
-    df["author"] = df["author"].apply(ast.literal_eval)
-    df = df[df["author"].apply(lambda x: any(item in members for item in x))]
-    df["author"] = df["author"].apply(
-        lambda x: [item for item in x if item not in nonmembers]
-    )
-    df["author"] = df["author"].apply(
-        lambda x: [item.replace("Côte DIvoire", "Cote D'Ivoire") for item in x]
-    )
-    df["draft_labs"] = df["draft_labs"].fillna("[]")
-    df["author"][
-        df["author"] == "['The Alliance Of Small Island States (AOSIS)']"
-    ] = "['Alliance Of Small Island States (AOSIS)']"
-
-    print(f"Filtered dataset to {len(df)} entries")
-    df.to_csv(DATASET_PROCESSED, index=False)
-
-
-if __name__ == "__main__":
-    main()

src/data_processing/document_store_data_all.py

@@ -1,93 +0,0 @@
-import pandas as pd
-import ast
-import json
-
-DATASET = "data/inc_df_v6_small.csv"
-DATASET_PROCESSED = "data/inc_df.csv"
-MEMBERS = "data/authors_filter.json"
-
-
-def main():
-    print(f"Length of dataset: {len(pd.read_csv(DATASET))}")
-    df = pd.read_csv(DATASET)
-    df["retriever_id"] = df.index
-    columns = [
-        "retriever_id",
-        "description",
-        "href",
-        "draft_labs_list",
-        "authors_list",
-        "draft_allcats",
-        "doc_subtype",
-        "doc_type",
-        "text",
-        "round",
-    ]
-
-    df = df[columns]
-
-    df.rename(
-        mapper={
-            "draft_labs_list": "draft_labs",
-            "draft_allcats": "draft_cats",
-            "authors_list": "author",
-        },
-        axis=1,
-        inplace=True,
-    )
-
-    ###Subselect for countries and country groups
-    with open(MEMBERS, "r") as f:
-        authors = json.load(f)
-    special_character_words_mapper = {
-        "Côte D'Ivoire": "Côte DIvoire",
-        "Ligue Camerounaise Des Droits De L'Homme": "Ligue Camerounaise Des Droits De LHomme",
-        "Association Pour L'Integration Et La Developpement Durable Au Burundi": "Association Pour LIntegration Et La Developpement Durable Au Burundi",
-    }
-    members = [
-        authors[key]
-        for key in [
-            "Members - Countries",
-            "Members - International and Regional State Associations",
-        ]
-    ]
-    members = [item for sublist in members for item in sublist]
-    members = [special_character_words_mapper.get(member, member) for member in members]
-
-    nonmembers = [
-        authors[key]
-        for key in [
-            "Intergovernmental Negotiation Committee",
-            "Observers and Other Participants",
-        ]
-    ]
-    nonmembers = [item for sublist in nonmembers for item in sublist]
-
-    df["author"][df["author"] == "['Côte D'Ivoire']"] = "['Côte DIvoire']"
-    df["author"][
-        df["author"] == "['Ligue Camerounaise Des Droits De L'Homme']"
-    ] = "['Ligue Camerounaise Des Droits De LHomme']"
-    df["author"][
-        df["author"]
-        == "['Association Pour L'Integration Et La Developpement Durable Au Burundi']"
-    ] = "['Association Pour LIntegration Et La Developpement Durable Au Burundi']"
-
-    df["author"] = df["author"].apply(ast.literal_eval)
-    df = df[df["author"].apply(lambda x: any(item in members for item in x))]
-    df["author"] = df["author"].apply(
-        lambda x: [item for item in x if item not in nonmembers]
-    )
-    df["author"] = df["author"].apply(
-        lambda x: [item.replace("Côte DIvoire", "Côte D 'Ivoire") for item in x]
-    )
-    df["draft_labs"] = df["draft_labs"].fillna("[]")
-    df["author"][
-        df["author"] == "['The Alliance Of Small Island States (AOSIS)']"
-    ] = "['Alliance Of Small Island States (AOSIS)']"
-
-    print(f"Filtered dataset to {len(df)} entries")
-    df.to_csv(DATASET_PROCESSED, index=False)
-
-
-if __name__ == "__main__":
-    main()

src/data_processing/get_meta_data_filter.py

@@ -1,21 +0,0 @@
-import pandas as pd
-import sys
-import os
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
-from src.rag.pipeline import RAGPipeline
-
-DATASET = os.path.join("data", "inc_df.csv")
-META_DATA = os.path.join("database", "meta_data.csv")
-
-rag_pipeline = RAGPipeline(
-    embedding_model="sentence-transformers/distiluse-base-multilingual-cased-v1",
-    prompt_template="src/rag/prompt_template.yaml",
-)
-
-meta_data = pd.DataFrame(
-    [document.meta for document in rag_pipeline.document_store.get_all_documents()]
-)
-
-meta_data = meta_data.drop_duplicates(subset=["retriever_id"], keep="first")
-
-meta_data.to_csv(META_DATA, index=False)

src/data_processing/taxonomy_processing.py

@@ -1,47 +0,0 @@
-import os
-
-from src.utils.data import load_json, save_json
-
-AUTHORS_TAXONOMY = os.path.join("data", "authors_taxonomy.json")
-AUTHORS_FILTER = os.path.join("data", "authors_filter.json")
-
-DRAFT_CATEGORIES_TAXONOMY = os.path.join("data", "draftcat_taxonomy.json")
-DRAFT_CATEGORIES_FILTER = os.path.join("data", "draftcat_taxonomy_filter.json")
-
-
-def get_authors(taxonomy: dict) -> dict:
-    countries = taxonomy["Members"]["Countries"]
-    associations = taxonomy["Members"][
-        "International and Regional State Associations"
-    ]  # noqa: E501
-    intergovernmental_negotiations = taxonomy[
-        "Intergovernmental Negotiation Committee"
-    ]  # noqa: E501
-    observers = taxonomy["Observers and Other Participants"]  # noqa: E501
-    return {
-        "Members - Countries": countries,
-        "Members - International and Regional State Associations": associations,  # noqa: E501
-        "Intergovernmental Negotiation Committee": intergovernmental_negotiations,  # noqa: E501
-        "Observers and Other Participants": observers,
-    }
-
-
-def get_draftcategories(taxonomy: dict) -> dict:
-    taxonomy_filter = {}
-    for draft_part, part_values in taxonomy.items():
-        part = draft_part
-        temp_values = []
-        for part_name, part_value in part_values.items():
-            temp_values.append(part_value)
-        taxonomy_filter[part] = temp_values
-    return taxonomy_filter
-
-
-if __name__ == "__main__":
-    authors_taxonomy = load_json(AUTHORS_TAXONOMY)
-    authors_filter = get_authors(authors_taxonomy)
-    save_json(file_path=AUTHORS_FILTER, data=authors_filter)
-
-    draft_categories_taxonomy = load_json(DRAFT_CATEGORIES_TAXONOMY)
-    draft_categories_filter = get_draftcategories(draft_categories_taxonomy)
-    save_json(file_path=DRAFT_CATEGORIES_FILTER, data=draft_categories_filter)

src/document_store/document_store.py

@@ -1,180 +0,0 @@
-from haystack.document_stores import InMemoryDocumentStore
-import pandas as pd
-import os
-import pathlib
-import ast
-from sklearn.preprocessing import MultiLabelBinarizer
-from langchain_community.document_loaders import DataFrameLoader
-from langchain.text_splitter import (
-    RecursiveCharacterTextSplitter,
-)
-from typing import Any
-
-
-INC_TEST_DATASET_PATH = os.path.join("data", "inc_df.csv")
-EMBEDDING_DIMENSION = 512
-
-special_character_words_mapper = {
-    "Côte D'Ivoire": "Côte DIvoire",
-    "Ligue Camerounaise Des Droits De L'Homme": "Ligue Camerounaise Des Droits De LHomme",
-    "Association Pour L'Integration Et La Developpement Durable Au Burundi": "Association Pour LIntegration Et La Developpement Durable Au Burundi",
-}
-special_character_words_reverse_mapper = {}
-for key, value in special_character_words_mapper.items():
-    special_character_words_reverse_mapper[value] = key
-
-
-def transform_to_list(row):
-    special_characters = False
-    if str(row) == "[]" or str(row) == "nan":
-        return []
-    else:
-        # replace special characters
-        for key, value in special_character_words_mapper.items():
-            if key in row:
-                row = row.replace(key, value)
-                special_characters = True
-        row = ast.literal_eval(row)
-        if special_characters:
-            for key, value in special_character_words_reverse_mapper.items():
-                if key in row:
-                    # get the index of the special character word
-                    index = row.index(key)
-                    # replace the special character word with the original word
-                    row[index] = value
-        return row
-
-
-def transform_data(df: pd.DataFrame):
-    # df["author"] = df["authors"].drop(columns=["authors"], axis=1)
-    df = df[df["doc_subtype"] != "Working documents"]
-    df = df[df["doc_subtype"] != "Contact Groups"]
-    df = df[df["doc_subtype"] != "Unsolicitated Submissions"]
-    df = df[df["doc_type"] != "official document"]
-    df = df[df["doc_subtype"] != "Stakeholder Dialogue"]
-    df["text"] = df["text"].astype(str).str.replace("_x000D_", " ")
-    df["text"] = df["text"].astype(str).str.replace("\n", " ")
-    # df["text"] = df["text"].astype(str).str.replace("\r", " ")
-    df["author"] = df["author"].str.replace("\xa0", " ")
-    df["author"] = df["author"].str.replace("ü", "u")
-    df["author"] = df["author"].str.strip()
-    df["author"] = df["author"].astype(str).str.replace("\r", " ")
-
-    df = df[
-        [
-            "author",
-            "doc_type",
-            "round",
-            "text",
-            "href",
-            "draft_labs",
-            "draft_cats",
-            "retriever_id",
-        ]
-    ].copy()
-
-    df = df.rename(columns={"text": "page_content"}).copy()
-
-    df["draft_labs2"] = df["draft_labs"]
-    df["author2"] = df["author"]
-
-    df["draft_labs"] = df.apply(lambda x: transform_to_list(x["draft_labs"]), axis=1)
-    df["author"] = df.apply(lambda x: transform_to_list(x["author"]), axis=1)
-
-    # df["draft_labs"] = df["draft_labs"].apply(
-    #     lambda x: ast.literal_eval(x) if str(x) != "[]" or str(x) != "nan" else []
-    # )
-    # df["author"] = df["author"].apply(
-    #     lambda x: ast.literal_eval(x) if str(x) != "[]" else []
-    # )
-
-    mlb = MultiLabelBinarizer(sparse_output=True)
-    mlb = MultiLabelBinarizer()
-    df = df.join(
-        pd.DataFrame(
-            mlb.fit_transform(df.pop("draft_labs")),
-            columns=mlb.classes_,
-            index=df.index,
-        )
-    ).join(
-        pd.DataFrame(
-            mlb.fit_transform(df.pop("author")), columns=mlb.classes_, index=df.index
-        )
-    )
-
-    df["draft_labs"] = df["draft_labs2"]
-    df = df.drop(columns=["draft_labs2"], axis=1)
-
-    df["author"] = df["author2"]
-    df = df.drop(columns=["author2"], axis=1)
-
-    loader = DataFrameLoader(df, page_content_column="page_content")
-    docs = loader.load()
-    return docs
-
-
-def process_data(docs):
-
-    chunk_size = 512
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=int(chunk_size / 10),
-        add_start_index=True,
-        strip_whitespace=True,
-        separators=["\n\n", "\n", " ", ""],
-    )
-
-    docs_chunked = text_splitter.transform_documents(docs)
-
-    df = pd.DataFrame(docs_chunked, columns=["page_content", "metadata", "type"]).drop(
-        "type", axis=1
-    )
-    df["page_content"] = df["page_content"].astype(str)
-    df["page_content"] = df["page_content"].str.replace("'page_content'", "")
-    df["page_content"] = df["page_content"].str.replace("(", "")
-    df["page_content"] = df["page_content"].str.replace(")", "").str[1:]
-    df = pd.concat(
-        [df.drop("metadata", axis=1), df["metadata"].apply(pd.Series)], axis=1
-    )
-    df = df.rename(columns={0: "a", 1: "b"})
-    df = pd.concat([df.drop(["a", "b"], axis=1), df["b"].apply(pd.Series)], axis=1)
-
-    cols = ["author", "draft_labs"]
-    for c in cols:
-        df[c] = df[c].apply(
-            lambda x: "".join(x) if isinstance(x, (list, tuple)) else str(x)
-        )
-        chars = ["[", "]", "'"]
-        for g in chars:
-            df[c] = df[c].str.replace(g, "")
-
-    df["page_content"] = df["page_content"].astype(str).str.replace("\n", " ")
-    df["page_content"] = df["page_content"].astype(str).str.replace("\r", " ")
-
-    cols = ["author", "draft_labs", "page_content"]
-    df["page_content"] = df[cols].apply(lambda row: " | ".join(row.astype(str)), axis=1)
-    df = df.rename(columns={"page_content": "content"})
-
-    documents = []
-    for _, row in df.iterrows():
-        row_meta: dict[str, Any] = {}
-        for column in df.columns:
-            if column != "content":
-                if column == "retriever_id":
-                    row_meta[column] = str(row[column])
-                else:
-                    row_meta[column] = row[column]
-        documents.append({"content": row["content"], "meta": row_meta})
-    return documents
-
-
-def get_document_store():
-    df = pd.read_csv(INC_TEST_DATASET_PATH)
-    # df["retriever_id"] = [str(i) for i in range(len(df))]
-    pathlib.Path("database").mkdir(parents=True, exist_ok=True)
-    document_store = InMemoryDocumentStore(
-        embedding_field="embedding", embedding_dim=EMBEDDING_DIMENSION, use_bm25=False
-    )
-    docs = transform_data(df=df)
-    document_store.write_documents(process_data(docs=docs))
-    return document_store

src/rag/pipeline.py

@@ -1,93 +0,0 @@
-import os
-import pickle
-from typing import Any
-
-from dotenv import load_dotenv
-from haystack.nodes import (  # type: ignore
-    AnswerParser,
-    EmbeddingRetriever,
-    PromptNode,
-    PromptTemplate,
-)
-from haystack.pipelines import Pipeline
-
-from src.document_store.document_store import get_document_store
-
-load_dotenv()
-
-OPENAI_API_KEY = os.environ.get("OPEN_API_KEY")
-
-
-class RAGPipeline:
-    def __init__(
-        self,
-        embedding_model: str,
-        prompt_template: str,
-    ):
-        self.load_document_store()
-        self.embedding_model = embedding_model
-        self.prompt_template = prompt_template
-        self.retriever_node = self.generate_retriever_node()
-        self.prompt_node = self.generate_prompt_node()
-        self.update_embeddings()
-        self.pipe = self.build_pipeline()
-
-    def run(self, prompt: str, filters: dict) -> Any:
-        try:
-            result = self.pipe.run(query=prompt, params={"filters": filters})
-            return result
-        except Exception as e:
-            print(e)
-            return None
-
-    def build_pipeline(self):
-        pipe = Pipeline()
-        pipe.add_node(component=self.retriever_node, name="retriever", inputs=["Query"])
-        pipe.add_node(
-            component=self.prompt_node,
-            name="prompt_node",
-            inputs=["retriever"],
-        )
-        return pipe
-
-    def load_document_store(self):
-        if os.path.exists(os.path.join("database", "document_store.pkl")):
-            with open(
-                file=os.path.join("database", "document_store.pkl"), mode="rb"
-            ) as f:
-                self.document_store = pickle.load(f)
-        else:
-            self.document_store = get_document_store()
-
-    def generate_retriever_node(self):
-        retriever_node = EmbeddingRetriever(
-            document_store=self.document_store,
-            embedding_model=self.embedding_model,
-            top_k=7,
-        )
-        return retriever_node
-
-    def update_embeddings(self):
-        if not os.path.exists(os.path.join("database", "document_store.pkl")):
-            self.document_store.update_embeddings(
-                self.retriever_node, update_existing_embeddings=True
-            )
-
-            with open(
-                file=os.path.join("database", "document_store.pkl"), mode="wb"
-            ) as f:
-                pickle.dump(self.document_store, f)
-
-    def generate_prompt_node(self):
-        rag_prompt = PromptTemplate(
-            prompt=self.prompt_template,
-            output_parser=AnswerParser(reference_pattern=r"Document\[(\d+)\]"),
-        )
-        prompt_node = PromptNode(
-            model_name_or_path="gpt-4",
-            default_prompt_template=rag_prompt,
-            api_key=OPENAI_API_KEY,
-            max_length=4000,
-            model_kwargs={"temperature": 0.2, "max_tokens": 4096},
-        )
-        return prompt_node

src/rag/prompt_template.yaml

@@ -1,20 +0,0 @@
-name: deepset/question-answering-with-document-references
-text: |
-  Answer the question '{query}' using only the provided documents and avoiding text. 
-  Formulate your answer in the style of an academic report. 
-  Provide example quotes and citations using extracted text from the documents.
-  Use facts and numbers from the documents in your answer. 
-  ALWAYS include the references of the documents used from documents at the end of each applicable sentence using the format [number].
-  If the answer isn't in the document say 'Answering is not possible given the available information'.
-  Documents: \n
-  {join(documents, delimiter=new_line, pattern=new_line+'Document($retriever_id): $content', str_replace={new_line: ' ', '[': '(', ']': ')'})} \n
-  Answer: 
-tags:
-  - question-answering
-description: Perform question answering with references to documents.
-meta:
-  authors:
-    - deepset-ai
-version: '0.1.0'
-
-

src/utils/data.py

@@ -1,76 +0,0 @@
-import json
-from typing import Any
-
-import pandas as pd
-import streamlit as st
-from functools import reduce
-
-
-def get_filter_values(df: pd.DataFrame, column_name: str) -> list:
-    return df[column_name].unique().tolist()
-
-
-def build_filter(
-    meta_data: pd.DataFrame,
-    authors_filter: list[str],
-    draft_cats_filter: list[str],
-    round_filter: list[int],
-) -> dict[str, int | str] | dict:
-    authors = authors_filter
-    round_number = round_filter
-    draft_cats = draft_cats_filter
-
-    # set authors_flag to True if not empty list
-    authors_flag = True if len(authors) > 0 else False
-    draft_cats_flag = True if len(draft_cats) > 0 else False
-    round_number_flag = True if len(round_number) > 0 else False
-
-    conditions = []
-
-    if authors_flag:
-        authors_condition = (meta_data[col] == 1 for col in authors)
-        authors_conditions_list = reduce(lambda a, b: a | b, authors_condition)
-        conditions.append(authors_conditions_list)
-
-    if draft_cats_flag:
-        draft_cat_condition = (meta_data[col] for col in draft_cats)
-        draft_cat_conditions_list = reduce(lambda a, b: a | b, draft_cat_condition)
-        conditions.append(draft_cat_conditions_list)
-
-    if round_number_flag:
-        round_condition = meta_data["round"].isin(round_number)
-        conditions.append(round_condition)
-
-    if len(conditions) == 0:
-        filtered_retriever_ids = []
-    else:
-        final_condition = reduce(lambda a, b: a & b, conditions)
-        filtered_retriever_ids = meta_data[final_condition]["retriever_id"].tolist()
-    if len(filtered_retriever_ids) == 0:
-        return {}
-    else:
-        return {"retriever_id": filtered_retriever_ids}
-
-
-def load_json(file_path: str) -> dict:
-    with open(file_path, "r") as f:
-        return json.load(f)
-
-
-def save_json(file_path: str, data: dict) -> None:
-    with open(file_path, "w") as f:
-        json.dump(data, f, indent=4)
-
-
-def get_meta(result: dict[str, Any]) -> list[dict[str, Any]]:
-    meta_data = []
-    for doc in result["documents"]:
-        current_meta = doc.meta
-        current_meta["content"] = doc.content
-        meta_data.append(current_meta)
-    return meta_data
-
-
-def load_css(file_name) -> None:
-    with open(file_name) as f:
-        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)

src/utils/data_v2.py

@@ -1,80 +0,0 @@
-import json
-from typing import Any
-
-import pandas as pd
-import streamlit as st
-from functools import reduce
-
-
-def get_filter_values(df: pd.DataFrame, column_name: str) -> list:
-    return df[column_name].unique().tolist()
-
-
-def build_filter(
-    meta_data: pd.DataFrame,
-    authors_filter: list[str],
-    draft_cats_filter: list[str],
- #   round_filter: list[int],
-) -> dict[str, int | str] | dict:
-    authors = authors_filter
-    #round_number = round_filter
-    draft_cats = draft_cats_filter
-
-    # set authors_flag to True if not empty list
-    authors_flag = True if len(authors) > 0 else False
-    draft_cats_flag = True if len(draft_cats) > 0 else False
-    #round_number_flag = True if len(round_number) > 0 else False
-
-    if authors_flag is False and draft_cats_flag is False:
-        return {}, "no filters selected"
-
-    conditions = []
-
-    if authors_flag:
-        authors_condition = (meta_data[col] == 1 for col in authors)
-        authors_conditions_list = reduce(lambda a, b: a | b, authors_condition)
-        conditions.append(authors_conditions_list)
-
-    if draft_cats_flag:
-        draft_cat_condition = (meta_data[col] for col in draft_cats)
-        draft_cat_conditions_list = reduce(lambda a, b: a | b, draft_cat_condition)
-        conditions.append(draft_cat_conditions_list)
-
-    # if round_number_flag:
-    #     round_condition = meta_data["round"].isin(round_number)
-    #     conditions.append(round_condition)
-
-    if len(conditions) == 0:
-        filtered_retriever_ids = []
-    else:
-        final_condition = reduce(lambda a, b: a & b, conditions)
-        filtered_retriever_ids = meta_data[final_condition]["retriever_id"].tolist()
-
-    if len(filtered_retriever_ids) == 0:
-        return {}, "no results found"
-    else:
-        return {"retriever_id": filtered_retriever_ids}, "success"
-
-
-def load_json(file_path: str) -> dict:
-    with open(file_path, "r") as f:
-        return json.load(f)
-
-
-def save_json(file_path: str, data: dict) -> None:
-    with open(file_path, "w") as f:
-        json.dump(data, f, indent=4)
-
-
-def get_meta(result: dict[str, Any]) -> list[dict[str, Any]]:
-    meta_data = []
-    for doc in result["documents"]:
-        current_meta = doc.meta
-        current_meta["content"] = doc.content
-        meta_data.append(current_meta)
-    return meta_data
-
-
-def load_css(file_name) -> None:
-    with open(file_name) as f:
-        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)

src/utils/writer.py

@@ -1,17 +0,0 @@
-import streamlit as st
-import time
-
-
-def typewriter(text: str, references: list, speed: int):
-    tokens = text.split()
-    container = st.empty()
-    for index in range(len(tokens) + 1):
-        curr_full_text = " ".join(tokens[:index])
-        container.markdown(curr_full_text)
-        time.sleep(1 / speed)
-    curr_full_text += "\n"
-    container.markdown(curr_full_text)
-    curr_full_text += "\n **References** \n"
-    container.markdown(curr_full_text)
-    curr_full_text += "\n".join(references)
-    container.markdown(curr_full_text)