{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import psycopg2\n", "\n", "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", "from langchain_community.document_loaders import DataFrameLoader\n", "from langchain_community.embeddings import HuggingFaceEmbeddings\n", "from langchain_community.vectorstores import FAISS\n", "from datetime import datetime\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Retrieve Speeches" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# db_connection -----------------------------------------------------------\n", "con_details = {\n", " \"host\" : \"localhost\",\n", " \"database\" : \"next\",\n", " \"user\" : \"postgres\",\n", " \"password\" : \"postgres\",\n", " \"port\" : \"5433\"\n", "}\n", "con = psycopg2.connect(**con_details)\n", "\n", "# get data tables ---------------------------------------------------------\n", "df = pd.read_sql_query(\"\"\"SELECT s.id,s.speech_content,s.date,f.abbreviation AS party\n", " FROM open_discourse.speeches AS s\n", " INNER JOIN open_discourse.factions AS f ON\n", " s.faction_id = f.id;\"\"\", con)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Process speeches" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(set(df['party'].to_list()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Removing keys from interruptions of a speech\n", "df[\"speech_content\"].replace(\"\\({\\d+}\\)\", \"\", inplace=True, regex=True) \n", "df['date'] = pd.to_datetime(df['date'])\n", "df" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idspeech_contentdateparty
00Meine Damen und Herren! Ich eröffne die 2. Sit...1949-09-12not found
11Der Bundesrat ist versammelt, Herr Präsident.\\n1949-09-12not found
22Ich danke für diese Erklärung. Ich stelle dami...1949-09-12not found
33Ja, ich habe den Wunsch.\\n1949-09-12not found
44Ich erteile dem Herrn Bundespräsidenten das Wo...1949-09-12not found
...............
9309551084268\\n\\nWir sind zwar Kollegen.2022-12-16not found
9309561084269\\n\\nLiebe, sehr geehrte Frau Präsidentin!2022-12-16CDU/CSU
9309571084270\\n\\nVielen Dank.2022-12-16not found
9309581084272\\n\\nDen Abschluss dieser Aktuellen Stunde bild...2022-12-16not found
9309591084273\\n\\nSehr geehrte Frau Präsidentin! Werte Kolle...2022-12-16SPD
\n", "

930960 rows × 4 columns

\n", "
" ], "text/plain": [ " id speech_content date \\\n", "0 0 Meine Damen und Herren! Ich eröffne die 2. Sit... 1949-09-12 \n", "1 1 Der Bundesrat ist versammelt, Herr Präsident.\\n 1949-09-12 \n", "2 2 Ich danke für diese Erklärung. Ich stelle dami... 1949-09-12 \n", "3 3 Ja, ich habe den Wunsch.\\n 1949-09-12 \n", "4 4 Ich erteile dem Herrn Bundespräsidenten das Wo... 1949-09-12 \n", "... ... ... ... \n", "930955 1084268 \\n\\nWir sind zwar Kollegen. 2022-12-16 \n", "930956 1084269 \\n\\nLiebe, sehr geehrte Frau Präsidentin! 2022-12-16 \n", "930957 1084270 \\n\\nVielen Dank. 2022-12-16 \n", "930958 1084272 \\n\\nDen Abschluss dieser Aktuellen Stunde bild... 2022-12-16 \n", "930959 1084273 \\n\\nSehr geehrte Frau Präsidentin! Werte Kolle... 2022-12-16 \n", "\n", " party \n", "0 not found \n", "1 not found \n", "2 not found \n", "3 not found \n", "4 not found \n", "... ... \n", "930955 not found \n", "930956 CDU/CSU \n", "930957 not found \n", "930958 not found \n", "930959 SPD \n", "\n", "[930960 rows x 4 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Convert to proper time format\n", "df['date'] = pd.to_datetime(df['date'])\n" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "def split_documents(df, min_chunk_size=100):\n", " \"\"\"\n", " Load documents from a DataFrame, split them into smaller chunks for vector storage and remove chunks of small size.\n", "\n", " Parameters\n", " ----------\n", " df : pandas.DataFrame\n", " A DataFrame containing the documents to be processed, with a column named 'speech_content'.\n", " min_chunk_size : int, optional\n", " Minimum number of characters a chunk must have to be included in the result. Default is 100.\n", "\n", " Returns\n", " -------\n", " list\n", " A list of split document chunks ready for further processing or vectorization.\n", " \"\"\"\n", " # Initialize a DataFrameLoader with the given DataFrame and specify the column containing the content to load\n", " loader = DataFrameLoader(data_frame=df, page_content_column='speech_content')\n", " # Load the data from the DataFrame into a suitable format for processing\n", " data = loader.load()\n", " # Initialize a RecursiveCharacterTextSplitter to split the text into chunks\n", " splitter = RecursiveCharacterTextSplitter(\n", " chunk_size=1024,\n", " chunk_overlap=32,\n", " length_function=len,\n", " is_separator_regex=False,\n", " )\n", " # Split the loaded data into smaller chunks using the splitter\n", " documents = splitter.split_documents(documents=data)\n", " # Discard small chunks below the threshold\n", " cleaned_documents = [doc for doc in documents if len(doc.page_content) >= min_chunk_size]\n", "\n", " return cleaned_documents" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "c:\\Python\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Sucessfully created vector store for 1. legislature\n", "Sucessfully created vector store for 2. legislature\n", "Sucessfully created vector store for 3. legislature\n", "Sucessfully created vector store for 4. legislature\n", "Sucessfully created vector store for 5. legislature\n", "Sucessfully created vector store for 6. legislature\n", "Sucessfully created vector store for 7. legislature\n", "Sucessfully created vector store for 8. legislature\n", "Sucessfully created vector store for 9. legislature\n", "Sucessfully created vector store for 10. legislature\n", "Sucessfully created vector store for 11. legislature\n", "Sucessfully created vector store for 12. legislature\n", "Sucessfully created vector store for 13. legislature\n", "Sucessfully created vector store for 14. legislature\n", "Sucessfully created vector store for 15. legislature\n", "Sucessfully created vector store for 16. legislature\n", "Sucessfully created vector store for 17. legislature\n", "Sucessfully created vector store for 18. legislature\n", "Sucessfully created vector store for 19. legislature\n", "Sucessfully created vector store for 20. legislature\n" ] } ], "source": [ "# Define starting dates of legislature periods\n", "dates = [\"1953-10-06\", \"1957-10-16\", \"1961-10-17\", \"1965-10-19\", \"1969-10-20\", \"1972-12-13\", \"1976-12-14\", \"1980-11-04\", \"1983-03-29\", \"1987-02-18\",\"1990-12-20\", \"1994-11-10\", \"1998-10-26\", \"2002-10-17\", \"2005-10-18\", \"2009-10-27\", \"2013-10-22\",\"2017-10-24\",\"2021-10-26\", None]\n", "# Load sentence transformer \n", "embeddings = HuggingFaceEmbeddings(model_name=\"paraphrase-multilingual-MiniLM-L12-v2\")\n", "\n", "# Create vector store for all speaches\n", "# Split text into documents for vectorstore\n", "documents = split_documents(df)\n", "# Create and save faiss vectorstorage\n", "index_name = 'speeches_1949_09_12'\n", "db = FAISS.from_documents(documents, embeddings)\n", "db.save_local(folder_path=\"FAISS\", index_name=index_name)\n", "print(\"Sucessfully created vector store for all legislature\")\n", "\n", "# Create vector store for each legislature\n", "# loop parameters\n", "period = 1\n", "previous_date = None\n", "\n", "# Iterate over all date to split by legislature getting vector stores for each period\n", "for date in dates:\n", " if previous_date is None:\n", " legislature_df = df.loc[df['date'] < datetime.strptime(date, \"%Y-%m-%d\")]\n", " elif date is None:\n", " legislature_df = df.loc[df['date'] >= datetime.strptime(previous_date, \"%Y-%m-%d\")]\n", " else:\n", " legislature_df = df.loc[(df['date'] >= datetime.strptime(previous_date, \"%Y-%m-%d\")) & (df['date'] < datetime.strptime(date, \"%Y-%m-%d\"))]\n", "\n", " \n", " # Split text into documents for vectorstore\n", " documents = split_documents(legislature_df)\n", "\n", " # Create and save faiss vectorstorage\n", " index_name = f'{period}_legislature'\n", " db = FAISS.from_documents(documents, embeddings)\n", " db.save_local(folder_path=\"FAISS\", index_name=index_name)\n", " print(f\"Sucessfully created vector store for {period}. legislature\")\n", "\n", " # Change loop parameters for next iteration\n", " period += 1\n", " previous_date = date\n", "\n", "\n", " \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This data has been uploaded to: https://huggingface.co/datasets/TomData/test" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 2 }