Upload data and app

app.py

@@ -0,0 +1,142 @@
+# Import packages
+import gradio as gr
+import pandas as pd
+from difflib import SequenceMatcher
+
+# Import results
+results_clean = pd.read_csv("data/retrieval_results/sonnini_cleaned/i_onit-sonnini-DHd2025-clean-q_Pferd, Pferde.csv").head(100)
+results_prep = pd.read_csv("data/retrieval_results/sonnini_llm_corrected/i_onit-sonnini-DHd2025-prep-q_Pferd, Pferde.csv").head(100)
+results_orig = pd.read_csv("data/retrieval_results/sonnini_original_ocr/i_onit-test-index-sonnini-q_Pferd-Pferde.csv").head(100)
+
+# Drop 'text_prep' from results_orig
+results_clean.drop(columns=['text_prep'], inplace=True)
+
+# Modify the "document" column to remove "_page175.txt" and keep the "Z166069305_00175"
+results_orig['document'] = results_orig['document'].str[:-12]
+
+# Modify the "page" column to extract the numeric part and remove leading zeroes
+results_orig['page'] = results_orig['page'].str.extract(r'(\d+)', expand=False).astype(int)
+
+data_sources = {"Results Cleaned OCR": results_clean, "Results LLM Preprocessed OCR": results_prep, "Results Original OCR": results_orig}
+
+# Pagination settings
+R = 5  # Number of preview rows per page
+
+# Define a function to highlight parts of the text
+def highlight_text(text, highlights):
+    # Ensure highlights is a list of strings
+    if isinstance(highlights, str):
+        highlights = [highlights]
+    # Wrap each highlight in <mark> tags
+    for highlight in highlights:
+        # Replace highlight text with a highlighted version
+        text = text.replace(highlight, f'<mark>{highlight}</mark>')
+    return text
+
+
+# Function to create preview rows
+def preview_results(page, selected_data_source):
+    data_source = data_sources[selected_data_source]
+    start_idx = (page - 1) * R
+    end_idx = min(start_idx + R, len(data_source))
+    
+    results = data_source.iloc[start_idx:end_idx]
+
+    row_elements = []
+    for idx, (_, row) in enumerate(results.iterrows(), start=start_idx + 1):
+        highlighted_text = row['unpacked_highlights']
+        row_html = f"""
+        <div style='border:1px solid #ddd; padding:10px; margin:5px 0; font-size: 18px;'>
+            <b>{idx}. \'{row['document']}\'</b> - Score: {row['_score']} - Rank: {row['rank']}
+            <br><i>{highlighted_text}</i>
+        </div>
+        """
+        row_elements.append(row_html)
+
+    return "".join(row_elements)
+
+# Function to show details of a selected row
+def show_details(document_name, selected_data_source):
+    data_source = data_sources[selected_data_source]
+    row = data_source[data_source["document"] == document_name]
+    
+    if row.empty:
+        return "<p style='color:red;'>Document not found. Please select a valid document.</p>"
+
+    row = row.iloc[0]  # Extract first matching row
+    return f"""
+    <div style="display: flex; justify-content: space-between; align-items: start;">
+        <div style="width: 65%; font-size: 18px;">
+            <h3>📄 Preview: {row['barcode']}, Page {row['page']}</h3>
+            <p><b>Retrieved text chunk: </b><i>{row["unpacked_highlights"]}</i></p>
+            <p><b>OCR text (LLM-corrected): </b>{highlight_text(row.get('text_prep') or row.get('text_clean') or row.get('text'), row["unpacked_highlights"])}</p>
+            <p><a href="https://digital.onb.ac.at/OnbViewer/viewer.faces?doc=ABO_%2B{row['barcode']}&order={row['page']}&view=SINGLE" target="_blank">🔍 Open ÖNB Viewer</a></p>
+        </div>
+        <div style="width: 30%; text-align: right;">
+            <img src="{row['iiif_link']}" alt="IIIF Image Preview" 
+                 style="max-width: 100%; height: auto; border: 1px solid #ddd;">
+        </div>
+    </div>
+    <div style="font-size: 18px;">
+        <p><b>Source: </b>C. S. Sonnini's, ehemaligen Offiziers und Jngenieurs des französischen Seewesens <br>und Mitgliedes mehrerer gelehrten und litterarischen Gesellschaften, <br><i>Reisen in Ober= und Niederägypten</i>, Bd. 1. Leipzig/Gera: Wilh. Heinsius, 1800</p>
+        <p><b>Citation link: <a href="http://data.onb.ac.at/rep/1058B194"target="_blank">http://data.onb.ac.at/rep/1058B194</a></p>
+    </div>
+    """
+
+# Gradio Interface
+with gr.Blocks() as demo:
+    gr.Markdown("## 🔍 Preview Text Retrieval Results with Marqo Vector Database")
+
+    data_source_dropdown = gr.Dropdown(choices=list(data_sources.keys()), label="Select Data Source", value="Results Cleaned OCR")
+    page_slider = gr.Slider(1, 1, step=1, label="Page", interactive=True)
+    preview_output = gr.HTML()
+
+    gr.Markdown("## 📝 Inspect Document Details")
+    
+    doc_name_input = gr.Textbox(label="Copy and paste document name to search bar (e.g. Z166069305_430):", interactive=True)
+    inspect_button = gr.Button("Inspect")
+    inspect_output = gr.HTML()
+
+    # Function to update preview when data source changes
+    def update_data_source(selected_data_source):
+        max_page = (len(data_sources[selected_data_source]) // R) + 1
+        page_slider.maximum = max_page  # Update the max page count dynamically
+        return preview_results(1, selected_data_source), 1  # Reset slider to 1
+
+    # Function to update preview when page slider changes
+    def update_preview(page, selected_data_source):
+        return preview_results(page, selected_data_source)
+
+    # Function to update document details
+    def update_details(doc_name, selected_data_source):
+        return show_details(doc_name, selected_data_source)
+
+    # Handle data source change
+    data_source_dropdown.change(
+        update_data_source,
+        inputs=[data_source_dropdown], 
+        outputs=[preview_output, page_slider]  # Update both preview and reset slider
+    )
+    # Handle page slider change
+    page_slider.change(update_preview, inputs=[page_slider, data_source_dropdown], outputs=[preview_output])
+
+    # Handle inspect button click
+    inspect_button.click(update_details, inputs=[doc_name_input, data_source_dropdown], outputs=[inspect_output])
+
+    # Initialize preview with default data source
+    preview_output.value = update_data_source("Results Cleaned OCR")
+    
+    # Further information block at the end
+    gr.Markdown("""
+    ## 📚 Further Information
+    <div style="font-size: 18px;">
+        <p>This demo lets you explore our preliminary results for retrieving <i>nature</i> representations in imperfect OCR data extracted from 17-19 century German texts.
+        This research was done in the <a href="https://onit.oeaw.ac.at/">Ottoman Nature in Travelogues (ONiT)</a> project and funded by the Austrian Science Fund (FWF: P 35245).
+        The text retrieval was done with hybrid vector/lexical search (BM25) by using a <a href="https://docs.marqo.ai/">Marqo</a> 
+        vector index. The texts were indexed as one page per document unit, and by splitting them in 2-sentence vectors and embedding them with 
+        <a href="https://huggingface.co/flax-sentence-embeddings/all_datasets_v4_mpnet-base">flax-sentence-embeddings/all_datasets_v4_mpnet-base</a> model.</p>
+        <p>For more information, contact <a href="mailto:[email protected]">michela(dot)vignoli(at)ait(dot)ac(dot)at</a>.</p>
+    </div>
+    """)
+
+demo.launch()

src/analysis/analysis_of_results.ipynb

@@ -0,0 +1,1515 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a5c7b436",
+   "metadata": {},
+   "source": [
+    "# Analysis of Retrieval Results"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6082ebb6",
+   "metadata": {},
+   "source": [
+    "## Load annotations and retrieval results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "fefb786d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>UUID</th>\n",
+       "      <th>FILE</th>\n",
+       "      <th>QUOTE_TRANSCRIPTION</th>\n",
+       "      <th>ANCHOR</th>\n",
+       "      <th>COMMENTS</th>\n",
+       "      <th>TAGS</th>\n",
+       "      <th>PAGE</th>\n",
+       "      <th>TRANSCRIPTION_CLEANED</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>551</th>\n",
+       "      <td>2faf6dce-7806-48b6-8e8b-2bf926fc88fc</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Negypten aber hat nicht allein Wunder der Kunſ...</td>\n",
+       "      <td>char-offset:2567</td>\n",
+       "      <td>p. 8</td>\n",
+       "      <td>'plants vegetation 25G'</td>\n",
+       "      <td>8</td>\n",
+       "      <td>('n? Negypten aber hat nicht allein Wunder der...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>552</th>\n",
+       "      <td>2e026f81-cbe7-43ba-9b37-5f5bd2760119</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Barbaren bewohnen jezt das Land und Räuber beh...</td>\n",
+       "      <td>char-offset:3391</td>\n",
+       "      <td>p. 8</td>\n",
+       "      <td>'plants vegetation 25G'</td>\n",
+       "      <td>8</td>\n",
+       "      <td>('r. Barbaren bewohnen jezt das Land und Raube...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>553</th>\n",
+       "      <td>0d5708db-1bf6-4de4-ad64-9759ebb4fa3c</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Die Waffen eines Mamefu den zu Pferde find zwe...</td>\n",
+       "      <td>char-offset:5538</td>\n",
+       "      <td>p. 9-10\\n</td>\n",
+       "      <td>'animals 25F', 'horses and kindred animals 46C...</td>\n",
+       "      <td>10</td>\n",
+       "      <td>('n. Die Waffen eines Mamefu den zu Pferde fin...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>554</th>\n",
+       "      <td>64b5828e-b14d-49dc-8d88-f97de52f9202</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Den Zaum reines Pferdes nimmt er zwiſchen die ...</td>\n",
+       "      <td>char-offset:5987</td>\n",
+       "      <td>p. 10\\n</td>\n",
+       "      <td>'animals 25F', 'horses and kindred animals 46C...</td>\n",
+       "      <td>10</td>\n",
+       "      <td>('l. Den Zaum reines Pferdes nimmt er zwischen...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>555</th>\n",
+       "      <td>fa2e8b0d-d8aa-46c8-ac97-1a8e2b14600d</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Seine Häuſer ſind arms felige Kothhütten, und ...</td>\n",
+       "      <td>char-offset:8769</td>\n",
+       "      <td>p. 11-12</td>\n",
+       "      <td>'plants vegetation 25G'</td>\n",
+       "      <td>12</td>\n",
+       "      <td>(' Hauser sind arms felige Kothhutten, und fei...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1117</th>\n",
+       "      <td>3d4a21ad-ddd8-4d3a-b018-d7dbb9f186dc</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Bittt adit2: 001), ont ma Des Abends begaben ſ...</td>\n",
+       "      <td>char-offset:821475</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>'animals 25F', 'birds 25F3'</td>\n",
+       "      <td>504</td>\n",
+       "      <td>(\"00 Bittt adit2: 001), ont ma Des Abends bega...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1118</th>\n",
+       "      <td>2aefbba4-e796-4add-bdb1-0768afe0a3ce</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>57 3 Chazelle lieb Chazelles</td>\n",
+       "      <td>char-offset:823734</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>'animals 25F', 'hoofed animalGAZELLE 25F24'</td>\n",
+       "      <td>504</td>\n",
+       "      <td>('ue 57 3 Chazelle lieb Chazelles 65', 814073)</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1119</th>\n",
+       "      <td>f50b57c6-d751-4d41-8670-8b5254d964ec</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Ein' to svil, dulu UPUN Fahn irlichin( jiyi vo...</td>\n",
+       "      <td>char-offset:824751</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>'animals 25F'</td>\n",
+       "      <td>504</td>\n",
+       "      <td>(\"c. Ein' to suil, dulu UPUN Fahn irlichin( ji...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1120</th>\n",
+       "      <td>d8cfcc30-5f00-4681-826b-c5c7a87e768b</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Dez HENNÈI, Blüthen. Eine Blüthe ron natiiitic...</td>\n",
+       "      <td>char-offset:824901</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>'plants vegetation 25G'</td>\n",
+       "      <td>504</td>\n",
+       "      <td>('-- Dez HENNEI, Bluthen. Eine Bluthe ron nati...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1121</th>\n",
+       "      <td>bbf95b5f-40d1-476a-a392-2dabb1278c16</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "      <td>Riedel fee. Fig. 1, ATLÉ, A, Stamn.2.7weig. Fi...</td>\n",
+       "      <td>char-offset:825427</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>'plants vegetation 25G'</td>\n",
+       "      <td>504</td>\n",
+       "      <td>(\" 1 Riedel fee. Fig. 1, ATLE, A, Stamn.2.7wei...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>571 rows × 8 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                      UUID                   FILE  \\\n",
+       "551   2faf6dce-7806-48b6-8e8b-2bf926fc88fc  Z166069305_merged.txt   \n",
+       "552   2e026f81-cbe7-43ba-9b37-5f5bd2760119  Z166069305_merged.txt   \n",
+       "553   0d5708db-1bf6-4de4-ad64-9759ebb4fa3c  Z166069305_merged.txt   \n",
+       "554   64b5828e-b14d-49dc-8d88-f97de52f9202  Z166069305_merged.txt   \n",
+       "555   fa2e8b0d-d8aa-46c8-ac97-1a8e2b14600d  Z166069305_merged.txt   \n",
+       "...                                    ...                    ...   \n",
+       "1117  3d4a21ad-ddd8-4d3a-b018-d7dbb9f186dc  Z166069305_merged.txt   \n",
+       "1118  2aefbba4-e796-4add-bdb1-0768afe0a3ce  Z166069305_merged.txt   \n",
+       "1119  f50b57c6-d751-4d41-8670-8b5254d964ec  Z166069305_merged.txt   \n",
+       "1120  d8cfcc30-5f00-4681-826b-c5c7a87e768b  Z166069305_merged.txt   \n",
+       "1121  bbf95b5f-40d1-476a-a392-2dabb1278c16  Z166069305_merged.txt   \n",
+       "\n",
+       "                                    QUOTE_TRANSCRIPTION              ANCHOR  \\\n",
+       "551   Negypten aber hat nicht allein Wunder der Kunſ...    char-offset:2567   \n",
+       "552   Barbaren bewohnen jezt das Land und Räuber beh...    char-offset:3391   \n",
+       "553   Die Waffen eines Mamefu den zu Pferde find zwe...    char-offset:5538   \n",
+       "554   Den Zaum reines Pferdes nimmt er zwiſchen die ...    char-offset:5987   \n",
+       "555   Seine Häuſer ſind arms felige Kothhütten, und ...    char-offset:8769   \n",
+       "...                                                 ...                 ...   \n",
+       "1117  Bittt adit2: 001), ont ma Des Abends begaben ſ...  char-offset:821475   \n",
+       "1118                       57 3 Chazelle lieb Chazelles  char-offset:823734   \n",
+       "1119  Ein' to svil, dulu UPUN Fahn irlichin( jiyi vo...  char-offset:824751   \n",
+       "1120  Dez HENNÈI, Blüthen. Eine Blüthe ron natiiitic...  char-offset:824901   \n",
+       "1121  Riedel fee. Fig. 1, ATLÉ, A, Stamn.2.7weig. Fi...  char-offset:825427   \n",
+       "\n",
+       "       COMMENTS                                               TAGS  PAGE  \\\n",
+       "551        p. 8                            'plants vegetation 25G'     8   \n",
+       "552        p. 8                            'plants vegetation 25G'     8   \n",
+       "553   p. 9-10\\n  'animals 25F', 'horses and kindred animals 46C...    10   \n",
+       "554     p. 10\\n  'animals 25F', 'horses and kindred animals 46C...    10   \n",
+       "555    p. 11-12                            'plants vegetation 25G'    12   \n",
+       "...         ...                                                ...   ...   \n",
+       "1117        NaN                        'animals 25F', 'birds 25F3'   504   \n",
+       "1118        NaN        'animals 25F', 'hoofed animalGAZELLE 25F24'   504   \n",
+       "1119        NaN                                      'animals 25F'   504   \n",
+       "1120        NaN                            'plants vegetation 25G'   504   \n",
+       "1121        NaN                            'plants vegetation 25G'   504   \n",
+       "\n",
+       "                                  TRANSCRIPTION_CLEANED  \n",
+       "551   ('n? Negypten aber hat nicht allein Wunder der...  \n",
+       "552   ('r. Barbaren bewohnen jezt das Land und Raube...  \n",
+       "553   ('n. Die Waffen eines Mamefu den zu Pferde fin...  \n",
+       "554   ('l. Den Zaum reines Pferdes nimmt er zwischen...  \n",
+       "555   (' Hauser sind arms felige Kothhutten, und fei...  \n",
+       "...                                                 ...  \n",
+       "1117  (\"00 Bittt adit2: 001), ont ma Des Abends bega...  \n",
+       "1118     ('ue 57 3 Chazelle lieb Chazelles 65', 814073)  \n",
+       "1119  (\"c. Ein' to suil, dulu UPUN Fahn irlichin( ji...  \n",
+       "1120  ('-- Dez HENNEI, Bluthen. Eine Bluthe ron nati...  \n",
+       "1121  (\" 1 Riedel fee. Fig. 1, ATLE, A, Stamn.2.7wei...  \n",
+       "\n",
+       "[571 rows x 8 columns]"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import os\n",
+    "import re\n",
+    "\n",
+    "text_annotations_path = \"data/annotations/\"\n",
+    "filename = 'DHd2025_referenceReports_merged-cleaned__annotations.xlsx'\n",
+    "\n",
+    "annotations = pd.read_excel(os.path.join(text_annotations_path, filename))\n",
+    "\n",
+    "## Remove unwanted tags from list\n",
+    "unwanted_strings = ['pages', 'subheading', 'footnote', 'table of contents', 'footnote irrelevant', 'title', 'scan mistake', 'dictionary']\n",
+    "\n",
+    "# Remove rows where 'TAGS' contains any unwanted string\n",
+    "annotations = annotations[~annotations['TAGS'].str.contains('|'.join(unwanted_strings), case=False, na=False)].reset_index(drop=True)\n",
+    "annotations = annotations[annotations['FILE'].str.contains('Z166069305', case=False, na=False)]\n",
+    "\n",
+    "annotations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c6bff210",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of excluded sentences: 0\n",
+      "Number of annotated mentions: 48\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'id': '0d5708db-1bf6-4de4-ad64-9759ebb4fa3c',\n",
+       " 'annotated_text': \"('n. Die Waffen eines Mamefu den zu Pferde find zwei --- 00010_page10_cleaned.txt --- grose Flinten, die ihm seine Diener zur Seite nach: tragen und die er ein einziges mal losschiest. Da', 5402)\",\n",
+       " 'comment': 'p. 9-10\\n',\n",
+       " 'page': 10,\n",
+       " 'file': 'Z166069305_merged.txt'}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## Filter the DataFrame\n",
+    "filtered_df = annotations[annotations['TAGS'].str.contains('horse', na=False)]\n",
+    "# Count the number of rows before filtering short sentences\n",
+    "before_filter_count = len(filtered_df)\n",
+    "\n",
+    "# Filter sentences that are 5 words or longer in 'QUOTE_TRANSCRIPTION'\n",
+    "filtered_df = filtered_df[filtered_df['TRANSCRIPTION_CLEANED'].apply(lambda x: len(x.split()) >= 5)]\n",
+    "# Count the number of rows after filtering short sentences\n",
+    "after_filter_count = len(filtered_df)\n",
+    "excluded_count = before_filter_count - after_filter_count\n",
+    "print(f\"Number of excluded sentences: {excluded_count}\")\n",
+    "\n",
+    "mentions_dict = filtered_df.set_index('UUID').to_dict(orient='index')\n",
+    "\n",
+    "mentions = [{\"id\": key, \"annotated_text\": v['TRANSCRIPTION_CLEANED'], \"comment\": v['COMMENTS'], \"page\": v['PAGE'], \"file\": v['FILE']} for key, v in mentions_dict.items()]\n",
+    "\n",
+    "print(f\"Number of annotated mentions: {len(filtered_df)}\")\n",
+    "mentions[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "cc29b140",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Total annotations: 48\n",
+      "On total pages: 40\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Summary of annotations\n",
+    "\n",
+    "# Total annotations:\n",
+    "# Remove duplicates based on \"page\"\n",
+    "unique_annot = {entry[\"id\"]: entry for entry in mentions}.values()\n",
+    "\n",
+    "# Remove duplicates based on \"page\"\n",
+    "unique_pages_annot = {entry[\"page\"]: entry for entry in mentions}.values()\n",
+    "\n",
+    "# Extract the \"page\" value from each dictionary of unique pages found\n",
+    "unique_pages_list = [item['page'] for item in unique_pages_annot]\n",
+    "\n",
+    "print(f'Total annotations: {len(unique_annot)}')\n",
+    "print(f'On total pages: {len(unique_pages_annot)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bd863c3b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Load retrieval results with Marqo DHd2025\n",
+    "\n",
+    "## Pferd, Pferde\n",
+    "retr_orig = pd.read_csv('data/retrieval_results/sonnini_original_OCR/i_onit-test-index-sonnini-q_Pferd-Pferde.csv')\n",
+    "retr_prep = pd.read_csv('data/retrieval_results/sonnini_llm_corrected/i_onit-sonnini-DHd2025-prep-q_Pferd, Pferde.csv')\n",
+    "retr_clean = pd.read_csv('data/retrieval_results/sonnini_cleaned/i_onit-sonnini-DHd2025-clean-q_Pferd, Pferde.csv')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6b4a54bd",
+   "metadata": {},
+   "source": [
+    "## Evaluation of results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "296250aa",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "200\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'id': '01b71d37-9ead-4eb2-8504-d16cbd00e866',\n",
+       " 'text_document': 'befand mich in einiger Entfernung davon, und der Ueber- rest unserer Begleitung folgte in ziemlicher Weite nach. Ein Trupp Beduinen zu Pferde. brach auf einmal hinter den Mauern hervor. Ich konnte sie an. fanglich unter den Staubwolken, die sie erregten, nicht unterscheiden: als fie fich aber ausgebreitet hatten, erfannte ich sowohl die Ort als die Anzahl von Leuten, mit denen wir zu thun haben sollten. Ich wandte sogleich inein Pferd um, und mit diefem vortreflichen Reuter, auf welchem man mich nicht hatte einbohlen konnen, hatte ich bald meine Gefahrten erreicht, die auch von ihren Came len herab diese. Reuterei gewahr worden waren. Ich fand sie zu Fus und in geschlossene Glieder gestellt. Ich sturzte mich von meinem Pferde herab und munterte fie zu einenu muthigen Widerstande auf. Wir waren im Ganzen rechs Personen, unter welchen wir nur auf drei | rechnen konnten. Zwei Eingebohrne fonnten nicht viel helfen, und der Zeichner war noch jung und ohne Erfah- rung, und konnte keine Flinte losschiesen. ut Die Standhaftigkeit einer Handvou Menschen, die fich mitten in einer Sandflache allein befanden, und die von keiner Seite gedekt waren, liberraschte eine Schwa: dron von ungefahr hundert Beduinen: man kann daraus schliesen, wie wenig furchtbar solche Feinde find, deren Muth fich blos darauf beschrankt, das fie fich in zahlreichen Horden versammeln, um eine leichte Plundes rung zu veruben, und auf eine feige Art den kleinen Krieg zu fuhren. Ob fie gleich in starkem Galop auf uns zuge. ritten kamen, so machten sie doch plozlich, etwann hundert Schritte von uns, halt. Sie riefen uns zu, kein Feuer zu geben, und ich erwiederte ihnen, fie sollten nicht vors rucken. Sie blieben einige Augenblicke in einer Art von Ungewisheit, was sie thun souten, wahrend welcher wir fie unter einander berathschlagen sahen. Sie theilten sich',\n",
+       " 'text_vector': 'befand mich in einiger Entfernung davon, und der Ueber- rest unserer Begleitung folgte in ziemlicher Weite nach. Ein Trupp Beduinen zu Pferde.',\n",
+       " 'page': 430,\n",
+       " 'rerank': 1,\n",
+       " 'barcode': 'Z166069305',\n",
+       " 'onb_link': 'https://digital.onb.ac.at/OnbViewer/viewer.faces?doc=ABO_%2BZ166069305'}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## Analyse overlap of retrieved texts with annotated texts\n",
+    "\n",
+    "\"\"\"\n",
+    "In this notebook, we attempted to compare the heterogeneous texts that were annotated during the project\n",
+    "with the retrieval results by comparing n-word phrases from the annotated sentences with the retrieved\n",
+    "sentence vectors. This approach is experimental. An alternative evaluation by calculating the Levenshtein\n",
+    "distance between the annotated text vectors and the retrieved text vectors will be done in future.\n",
+    "\"\"\"\n",
+    "\n",
+    "## Create dict from retrieved texts\n",
+    "#retr = retr_orig.head(200)\n",
+    "#retr = retr_prep.head(200)\n",
+    "retr = retr_clean.head(200)\n",
+    "\n",
+    "retrieved_dict = retr.set_index('_id').to_dict(orient='index')\n",
+    "\n",
+    "retrieved = [\n",
+    "    {\n",
+    "        \"id\": key,\n",
+    "        \"text_document\": v['text_clean'], # 'text', 'text_clean'\n",
+    "        \"text_vector\": v['unpacked_highlights'] if pd.notna(v['unpacked_highlights']) else v['text'],\n",
+    "        \"page\": v['page'], #int((re.search(r'\\d+', v['page'])).group()),\n",
+    "        \"rerank\": v['rerank'],\n",
+    "        \"barcode\": v['barcode'],\n",
+    "        \"onb_link\": v['onb_viewer_link']\n",
+    "    }\n",
+    "    for key, v in retrieved_dict.items()\n",
+    "]\n",
+    "\n",
+    "print(len(retrieved))\n",
+    "retrieved[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "307ab5c6",
+   "metadata": {
+    "code_folding": []
+   },
+   "outputs": [],
+   "source": [
+    "## Function to generate n-word phrases from a sentence\n",
+    "def generate_n_word_phrases(sentence, n):\n",
+    "    words = sentence.lower().split()\n",
+    "    return [' '.join(words[i:i+n]) for i in range(len(words) - n + 1)]\n",
+    "\n",
+    "## Function to check if any of the n-word phrases is in the text_vector\n",
+    "## and store the first matching phrase\n",
+    "def check_sentence_in_text_vector(sentence, text_vector, n):\n",
+    "    text_vector_lower = text_vector.lower()\n",
+    "    phrases = generate_n_word_phrases(sentence, n)\n",
+    "    for phrase in phrases:\n",
+    "        if phrase in text_vector_lower:\n",
+    "            return True, phrase  # Return True and the matching phrase\n",
+    "    return False, None  # No match found, return None\n",
+    "    #return any(phrase in text_vector_lower for phrase in phrases)\n",
+    "\n",
+    "# Number of words in the phrase to check\n",
+    "n = 5  # Change this to the desired number of words\n",
+    "\n",
+    "# List to store results\n",
+    "results = []\n",
+    "\n",
+    "# Iterate through each sentence in the mentions list\n",
+    "for sentence in mentions:\n",
+    "    found = False\n",
+    "    # Check if any n-word phrase from this sentence is present in any of the text_vector fields\n",
+    "    for entry in retrieved:\n",
+    "        #print(entry['text_vector'])\n",
+    "        match_found, match = check_sentence_in_text_vector(sentence['annotated_text'], entry['text_vector'], n)\n",
+    "        if match_found == True:\n",
+    "            found = True\n",
+    "            break  # No need to check further if a match is found\n",
+    "    if found == True:\n",
+    "        results.append({\n",
+    "            \"sentence_annotation\": sentence['annotated_text'],\n",
+    "            \"id_annotation\": sentence['id'],\n",
+    "            \"page_annotation\": sentence['page'],\n",
+    "            \"comment\": sentence['comment'],\n",
+    "            \"found_in_text_vector\": found,\n",
+    "            \"text_vector\": entry['text_vector'],\n",
+    "            \"matching_phrase\": match,\n",
+    "            \"id_text_vector\": entry['id'],\n",
+    "            \"barcode\": entry['barcode'],\n",
+    "            \"page_text\": entry['page'],\n",
+    "            \"onb_link\": entry['onb_link'],\n",
+    "            \"file_annotation\": sentence['file']\n",
+    "            })\n",
+    "    else:\n",
+    "        results.append({\n",
+    "            \"sentence_annotation\": sentence['annotated_text'],\n",
+    "            \"id_annotation\": sentence['id'],\n",
+    "            \"page_annotation\": sentence['page'],\n",
+    "            \"file_annotation\": sentence['file'],\n",
+    "            \"found_in_text_vector\": found\n",
+    "            })\n",
+    "\n",
+    "retrieved_df = pd.DataFrame(retrieved)\n",
+    "retrieved_df.rename(columns={'id': 'id_text_vector'}, inplace=True)\n",
+    "mat_df = pd.DataFrame(results)\n",
+    "matches_df = retrieved_df.merge(mat_df, on='id_text_vector', how='outer')\n",
+    "\n",
+    "# Sort the DataFrame based on the 'rrf_rank' column in ascending order\n",
+    "matches_df = matches_df.sort_values(by='rerank', ascending=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "68868787",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True Positives: Found 27 relevant mentions on 27 pages of 48 relevant mentions on 40 pages\n",
+      "(67.50%).\n",
+      "Found average of 1.00 found mentions per page vs. GT of 1.20 annotations per page.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sentence_annotation</th>\n",
+       "      <th>text_vector_x</th>\n",
+       "      <th>rerank</th>\n",
+       "      <th>id_annotation</th>\n",
+       "      <th>page</th>\n",
+       "      <th>page_annotation</th>\n",
+       "      <th>file_annotation</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>('h. Ein Trupp Beduinen zu Pferde. brach auf e...</td>\n",
+       "      <td>befand mich in einiger Entfernung davon, und d...</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>ffbabe61-0a21-4e4f-b6e4-2ddc4724b40c</td>\n",
+       "      <td>430.0</td>\n",
+       "      <td>434.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>('n. Man muntert beide auch mit einem Klatsche...</td>\n",
+       "      <td>Man muntert beide auch mit einem Klatschen der...</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>9a41db07-a5c8-4b3f-931a-43418f800768</td>\n",
+       "      <td>399.0</td>\n",
+       "      <td>403.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>('t. Ein kriegerisches und eroberungssuchtiges...</td>\n",
+       "      <td>Eine weife Nation hingegen, die den Ackerbau f...</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>2caacee3-c22d-4a76-a2d7-ba794856a103</td>\n",
+       "      <td>175.0</td>\n",
+       "      <td>176.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>('n. Die Waffen eines Mamefu den zu Pferde fin...</td>\n",
+       "      <td>Die Waffen eines Mamefu den zu Pferde find zwei</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>0d5708db-1bf6-4de4-ad64-9759ebb4fa3c</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>('r. Das heise Clima dieser Gegenden des Alter...</td>\n",
+       "      <td>Das Pferd, das eben fo feurig ist als die Luft...</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>4f7c0172-b25f-4e62-9cb5-a1c1213c6d7d</td>\n",
+       "      <td>220.0</td>\n",
+       "      <td>222.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>(\"n. Diese Banditen waren nicht damit zufriede...</td>\n",
+       "      <td>Andere Uraber erwiesen meinen Gefahrten diesel...</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>c86544f5-33f0-4d9a-b048-4995ae36c493</td>\n",
+       "      <td>435.0</td>\n",
+       "      <td>439.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>('s. ?. Wir fahen zwei Beduinen zu Pferde; fie...</td>\n",
+       "      <td>?. Wir fahen zwei Beduinen zu Pferde; fie floh...</td>\n",
+       "      <td>7.0</td>\n",
+       "      <td>3a1cf968-2b82-48d9-a784-d3a7f5198890</td>\n",
+       "      <td>413.0</td>\n",
+       "      <td>417.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>(\"b. Der Hraber muste mir fur den taglichen Pr...</td>\n",
+       "      <td>Der Hraber muste mir fur den taglichen Preis v...</td>\n",
+       "      <td>8.0</td>\n",
+       "      <td>b0450149-d074-4079-87c4-cdc252ce7c67</td>\n",
+       "      <td>383.0</td>\n",
+       "      <td>387.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>('r. Ich war mit meinem Pferde uorausgeritten,...</td>\n",
+       "      <td>Ich war mit meinem Pferde vorausgeritten, und ...</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>a16c02e9-16a6-4780-b65e-51510b8db6fb</td>\n",
+       "      <td>406.0</td>\n",
+       "      <td>410.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>('-- Das Lager stand etwan einige hundert Fus ...</td>\n",
+       "      <td>Man bauete verschiedente Arten Viehfutter dara...</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>b8a8f0ea-926e-4d89-b21d-e89e69a35c2f</td>\n",
+       "      <td>394.0</td>\n",
+       "      <td>397.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>(\"n. Ich bemerkte oft frische Spuren uon Strau...</td>\n",
+       "      <td>Ihre Jagd ist eine von den Leibesubuns gen, wo...</td>\n",
+       "      <td>11.0</td>\n",
+       "      <td>2703eecd-2bb6-432b-beaf-c92933736058</td>\n",
+       "      <td>424.0</td>\n",
+       "      <td>428.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>('b. Ich bestieg ein sehr schones Pferd, und i...</td>\n",
+       "      <td>Ich bestieg ein sehr schones Pferd, und ich wu...</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>3f08cff7-7220-4c5e-9d1f-1b8d4fc92028</td>\n",
+       "      <td>405.0</td>\n",
+       "      <td>409.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>('n. Sie halten sich mit ihren trefa lich klei...</td>\n",
+       "      <td>Sie halten sich mit ihren trefa lich kleinen P...</td>\n",
+       "      <td>13.0</td>\n",
+       "      <td>28370a93-4521-48d0-a8da-50545822debc</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>('n. Es war eine sehr gunstige Gelegenheit: di...</td>\n",
+       "      <td>mittel herum, die man ihnen gereicht hatte: ih...</td>\n",
+       "      <td>14.0</td>\n",
+       "      <td>d9613671-efc9-4a0e-9770-d2b8432d71de</td>\n",
+       "      <td>455.0</td>\n",
+       "      <td>460.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>('r. Ihre ganze Nahrung besteht in Brodt oder ...</td>\n",
+       "      <td>Und wirklich sind diejenigen, die in der Wuste...</td>\n",
+       "      <td>15.0</td>\n",
+       "      <td>5913d181-6da7-4169-9187-9eb885d33e33</td>\n",
+       "      <td>448.0</td>\n",
+       "      <td>452.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>('g. Sie waren bald bei uns und hatten uns, oh...</td>\n",
+       "      <td>Es verdros mich, das ich mich solchen Raubern ...</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>1a1ef27f-0449-4664-8f6c-e929f089504a</td>\n",
+       "      <td>431.0</td>\n",
+       "      <td>435.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>('d. Der Efel, der den Munduorrath trug und ei...</td>\n",
+       "      <td>Der Efel, der den Mundvorrath trug und einer l...</td>\n",
+       "      <td>18.0</td>\n",
+       "      <td>5892d1ea-8016-4e3e-9176-f6f6df833125</td>\n",
+       "      <td>148.0</td>\n",
+       "      <td>149.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>('t. Man reitet zu Pferde durch dieselbe, wenn...</td>\n",
+       "      <td>Man reitet zu Pferde durch dieselbe, wenn nich...</td>\n",
+       "      <td>19.0</td>\n",
+       "      <td>e422911b-8c7b-4a1e-a4aa-df3d3f617c51</td>\n",
+       "      <td>152.0</td>\n",
+       "      <td>153.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>(\"n. Eine Viertelmeile weit richteten wir unse...</td>\n",
+       "      <td>Eine Viertelmeile weit richteten wir unsern We...</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>2602d32c-2c40-4add-893c-abfc115b3c38</td>\n",
+       "      <td>415.0</td>\n",
+       "      <td>419.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>(\"n. Auf den Strasen uerkaufte man Bokshornste...</td>\n",
+       "      <td>Auf den Strasen verkaufte man Bokshornstengel*...</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>b3a06b70-a367-4eef-a21e-788142390022</td>\n",
+       "      <td>261.0</td>\n",
+       "      <td>263.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>('l. Den Zaum reines Pferdes nimmt er zwischen...</td>\n",
+       "      <td>Sein letztes Hulfeitrittet find jwei Sa: bel. ...</td>\n",
+       "      <td>22.0</td>\n",
+       "      <td>64b5828e-b14d-49dc-8d88-f97de52f9202</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>('n. Es giebt ein gutes Futter, das die Pferde...</td>\n",
+       "      <td>machten Eigenschaften verdient der Anbau das B...</td>\n",
+       "      <td>23.0</td>\n",
+       "      <td>3bf200db-aeb9-4823-8923-6458b1f01d7e</td>\n",
+       "      <td>263.0</td>\n",
+       "      <td>266.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>('-- Wir kamen aus einer Stadt, wo die Menge u...</td>\n",
+       "      <td>Wir kamen aus einer Stadt, wo die Menge von Ma...</td>\n",
+       "      <td>25.0</td>\n",
+       "      <td>cdd8f310-f15e-44d7-bd06-3e91351e3eae</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>(\"n. Den Sag nach der Ankunft dieses Mannes en...</td>\n",
+       "      <td>Den Sag nach der Ankunft dieses Mannes entdekt...</td>\n",
+       "      <td>26.0</td>\n",
+       "      <td>59c9905e-6cfa-4544-8805-dfbd53e7ed29</td>\n",
+       "      <td>454.0</td>\n",
+       "      <td>459.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>('n. Die Araber uerfolgen fte zu Pferde; und m...</td>\n",
+       "      <td>Sie entfernen Fich aber sogleich, fo bald fie ...</td>\n",
+       "      <td>27.0</td>\n",
+       "      <td>63f2c3e0-69a8-4a22-a356-7d0dd570dee1</td>\n",
+       "      <td>422.0</td>\n",
+       "      <td>426.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>33</th>\n",
+       "      <td>(\"t. Jene artige Art uon uierfusigent Thieren,...</td>\n",
+       "      <td>brauch von ihnen zu machen. Diese Fufe' find w...</td>\n",
+       "      <td>34.0</td>\n",
+       "      <td>1f056aa1-5980-4f62-9bff-147a9b409fd9</td>\n",
+       "      <td>423.0</td>\n",
+       "      <td>427.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>62</th>\n",
+       "      <td>('t. Wir erkletterten auf Maulefeln die Sandhu...</td>\n",
+       "      <td>Wir erkletterten auf Maulefeln die Sandhugel, ...</td>\n",
+       "      <td>63.0</td>\n",
+       "      <td>e4074c9a-c71c-4076-a22e-dce3f7e263c9</td>\n",
+       "      <td>264.0</td>\n",
+       "      <td>267.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                  sentence_annotation  \\\n",
+       "0   ('h. Ein Trupp Beduinen zu Pferde. brach auf e...   \n",
+       "1   ('n. Man muntert beide auch mit einem Klatsche...   \n",
+       "2   ('t. Ein kriegerisches und eroberungssuchtiges...   \n",
+       "3   ('n. Die Waffen eines Mamefu den zu Pferde fin...   \n",
+       "4   ('r. Das heise Clima dieser Gegenden des Alter...   \n",
+       "5   (\"n. Diese Banditen waren nicht damit zufriede...   \n",
+       "6   ('s. ?. Wir fahen zwei Beduinen zu Pferde; fie...   \n",
+       "7   (\"b. Der Hraber muste mir fur den taglichen Pr...   \n",
+       "8   ('r. Ich war mit meinem Pferde uorausgeritten,...   \n",
+       "9   ('-- Das Lager stand etwan einige hundert Fus ...   \n",
+       "10  (\"n. Ich bemerkte oft frische Spuren uon Strau...   \n",
+       "11  ('b. Ich bestieg ein sehr schones Pferd, und i...   \n",
+       "12  ('n. Sie halten sich mit ihren trefa lich klei...   \n",
+       "13  ('n. Es war eine sehr gunstige Gelegenheit: di...   \n",
+       "14  ('r. Ihre ganze Nahrung besteht in Brodt oder ...   \n",
+       "15  ('g. Sie waren bald bei uns und hatten uns, oh...   \n",
+       "17  ('d. Der Efel, der den Munduorrath trug und ei...   \n",
+       "18  ('t. Man reitet zu Pferde durch dieselbe, wenn...   \n",
+       "19  (\"n. Eine Viertelmeile weit richteten wir unse...   \n",
+       "20  (\"n. Auf den Strasen uerkaufte man Bokshornste...   \n",
+       "21  ('l. Den Zaum reines Pferdes nimmt er zwischen...   \n",
+       "22  ('n. Es giebt ein gutes Futter, das die Pferde...   \n",
+       "24  ('-- Wir kamen aus einer Stadt, wo die Menge u...   \n",
+       "25  (\"n. Den Sag nach der Ankunft dieses Mannes en...   \n",
+       "26  ('n. Die Araber uerfolgen fte zu Pferde; und m...   \n",
+       "33  (\"t. Jene artige Art uon uierfusigent Thieren,...   \n",
+       "62  ('t. Wir erkletterten auf Maulefeln die Sandhu...   \n",
+       "\n",
+       "                                        text_vector_x  rerank  \\\n",
+       "0   befand mich in einiger Entfernung davon, und d...     1.0   \n",
+       "1   Man muntert beide auch mit einem Klatschen der...     2.0   \n",
+       "2   Eine weife Nation hingegen, die den Ackerbau f...     3.0   \n",
+       "3     Die Waffen eines Mamefu den zu Pferde find zwei     4.0   \n",
+       "4   Das Pferd, das eben fo feurig ist als die Luft...     5.0   \n",
+       "5   Andere Uraber erwiesen meinen Gefahrten diesel...     6.0   \n",
+       "6   ?. Wir fahen zwei Beduinen zu Pferde; fie floh...     7.0   \n",
+       "7   Der Hraber muste mir fur den taglichen Preis v...     8.0   \n",
+       "8   Ich war mit meinem Pferde vorausgeritten, und ...     9.0   \n",
+       "9   Man bauete verschiedente Arten Viehfutter dara...    10.0   \n",
+       "10  Ihre Jagd ist eine von den Leibesubuns gen, wo...    11.0   \n",
+       "11  Ich bestieg ein sehr schones Pferd, und ich wu...    12.0   \n",
+       "12  Sie halten sich mit ihren trefa lich kleinen P...    13.0   \n",
+       "13  mittel herum, die man ihnen gereicht hatte: ih...    14.0   \n",
+       "14  Und wirklich sind diejenigen, die in der Wuste...    15.0   \n",
+       "15  Es verdros mich, das ich mich solchen Raubern ...    16.0   \n",
+       "17  Der Efel, der den Mundvorrath trug und einer l...    18.0   \n",
+       "18  Man reitet zu Pferde durch dieselbe, wenn nich...    19.0   \n",
+       "19  Eine Viertelmeile weit richteten wir unsern We...    20.0   \n",
+       "20  Auf den Strasen verkaufte man Bokshornstengel*...    21.0   \n",
+       "21  Sein letztes Hulfeitrittet find jwei Sa: bel. ...    22.0   \n",
+       "22  machten Eigenschaften verdient der Anbau das B...    23.0   \n",
+       "24  Wir kamen aus einer Stadt, wo die Menge von Ma...    25.0   \n",
+       "25  Den Sag nach der Ankunft dieses Mannes entdekt...    26.0   \n",
+       "26  Sie entfernen Fich aber sogleich, fo bald fie ...    27.0   \n",
+       "33  brauch von ihnen zu machen. Diese Fufe' find w...    34.0   \n",
+       "62  Wir erkletterten auf Maulefeln die Sandhugel, ...    63.0   \n",
+       "\n",
+       "                           id_annotation   page  page_annotation  \\\n",
+       "0   ffbabe61-0a21-4e4f-b6e4-2ddc4724b40c  430.0            434.0   \n",
+       "1   9a41db07-a5c8-4b3f-931a-43418f800768  399.0            403.0   \n",
+       "2   2caacee3-c22d-4a76-a2d7-ba794856a103  175.0            176.0   \n",
+       "3   0d5708db-1bf6-4de4-ad64-9759ebb4fa3c    9.0             10.0   \n",
+       "4   4f7c0172-b25f-4e62-9cb5-a1c1213c6d7d  220.0            222.0   \n",
+       "5   c86544f5-33f0-4d9a-b048-4995ae36c493  435.0            439.0   \n",
+       "6   3a1cf968-2b82-48d9-a784-d3a7f5198890  413.0            417.0   \n",
+       "7   b0450149-d074-4079-87c4-cdc252ce7c67  383.0            387.0   \n",
+       "8   a16c02e9-16a6-4780-b65e-51510b8db6fb  406.0            410.0   \n",
+       "9   b8a8f0ea-926e-4d89-b21d-e89e69a35c2f  394.0            397.0   \n",
+       "10  2703eecd-2bb6-432b-beaf-c92933736058  424.0            428.0   \n",
+       "11  3f08cff7-7220-4c5e-9d1f-1b8d4fc92028  405.0            409.0   \n",
+       "12  28370a93-4521-48d0-a8da-50545822debc   12.0             12.0   \n",
+       "13  d9613671-efc9-4a0e-9770-d2b8432d71de  455.0            460.0   \n",
+       "14  5913d181-6da7-4169-9187-9eb885d33e33  448.0            452.0   \n",
+       "15  1a1ef27f-0449-4664-8f6c-e929f089504a  431.0            435.0   \n",
+       "17  5892d1ea-8016-4e3e-9176-f6f6df833125  148.0            149.0   \n",
+       "18  e422911b-8c7b-4a1e-a4aa-df3d3f617c51  152.0            153.0   \n",
+       "19  2602d32c-2c40-4add-893c-abfc115b3c38  415.0            419.0   \n",
+       "20  b3a06b70-a367-4eef-a21e-788142390022  261.0            263.0   \n",
+       "21  64b5828e-b14d-49dc-8d88-f97de52f9202   10.0             10.0   \n",
+       "22  3bf200db-aeb9-4823-8923-6458b1f01d7e  263.0            266.0   \n",
+       "24  cdd8f310-f15e-44d7-bd06-3e91351e3eae   59.0             59.0   \n",
+       "25  59c9905e-6cfa-4544-8805-dfbd53e7ed29  454.0            459.0   \n",
+       "26  63f2c3e0-69a8-4a22-a356-7d0dd570dee1  422.0            426.0   \n",
+       "33  1f056aa1-5980-4f62-9bff-147a9b409fd9  423.0            427.0   \n",
+       "62  e4074c9a-c71c-4076-a22e-dce3f7e263c9  264.0            267.0   \n",
+       "\n",
+       "          file_annotation  \n",
+       "0   Z166069305_merged.txt  \n",
+       "1   Z166069305_merged.txt  \n",
+       "2   Z166069305_merged.txt  \n",
+       "3   Z166069305_merged.txt  \n",
+       "4   Z166069305_merged.txt  \n",
+       "5   Z166069305_merged.txt  \n",
+       "6   Z166069305_merged.txt  \n",
+       "7   Z166069305_merged.txt  \n",
+       "8   Z166069305_merged.txt  \n",
+       "9   Z166069305_merged.txt  \n",
+       "10  Z166069305_merged.txt  \n",
+       "11  Z166069305_merged.txt  \n",
+       "12  Z166069305_merged.txt  \n",
+       "13  Z166069305_merged.txt  \n",
+       "14  Z166069305_merged.txt  \n",
+       "15  Z166069305_merged.txt  \n",
+       "17  Z166069305_merged.txt  \n",
+       "18  Z166069305_merged.txt  \n",
+       "19  Z166069305_merged.txt  \n",
+       "20  Z166069305_merged.txt  \n",
+       "21  Z166069305_merged.txt  \n",
+       "22  Z166069305_merged.txt  \n",
+       "24  Z166069305_merged.txt  \n",
+       "25  Z166069305_merged.txt  \n",
+       "26  Z166069305_merged.txt  \n",
+       "33  Z166069305_merged.txt  \n",
+       "62  Z166069305_merged.txt  "
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Filter rows where 'found_in_text_vector' is True --> True Positives\n",
+    "found = matches_df[matches_df['found_in_text_vector'] == True]\n",
+    "found_fil = found['id_annotation'].drop_duplicates(keep='first')\n",
+    "found_pages = found['page'].drop_duplicates(keep='first')\n",
+    "\n",
+    "# Extract the \"page\" value from each dictionary of unique pages found\n",
+    "unique_foundPages_list = found_pages.to_list()\n",
+    "\n",
+    "# Filter rows where the absolute difference between 'page_annotation' and 'page_text' is less than or equal to 4\n",
+    "#found = found[abs(found['page_annotation'] - found['page_text']) <= 6]\n",
+    "\n",
+    "print(f\"True Positives: Found {len(found_fil)} relevant mentions on {len(unique_foundPages_list)} pages of {len(unique_annot)} relevant mentions on {len(unique_pages_list)} pages\")\n",
+    "print(f\"({len(unique_foundPages_list)/len(unique_pages_list)*100:.2f}%).\")\n",
+    "print(f\"Found average of {len(found_fil)/len(found_pages):.2f} found mentions per page vs. GT of {len(unique_annot)/len(unique_pages_annot):.2f} annotations per page.\")\n",
+    "\n",
+    "found[['sentence_annotation', 'text_vector_x', 'rerank', 'id_annotation', 'page', 'page_annotation', 'file_annotation']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "1e9beb8e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "False Negatives: Missed 21 mentions on 20 pages, of which 0 pages were retrieved with another vector.\n",
+      "(50.00%).\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sentence_annotation</th>\n",
+       "      <th>id_annotation</th>\n",
+       "      <th>page_annotation</th>\n",
+       "      <th>file_annotation</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>200</th>\n",
+       "      <td>('r. Wir giengen in geschloffenen Reihen und m...</td>\n",
+       "      <td>a406cecb-48dc-4b27-9e2e-e831301a59bc</td>\n",
+       "      <td>149.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>201</th>\n",
+       "      <td>('s. Da die Wagen nicht gewohnlich sind, so be...</td>\n",
+       "      <td>53e38871-8353-4ea1-acc5-e60964d4b02d</td>\n",
+       "      <td>151.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>202</th>\n",
+       "      <td>('t. Man tritt mit den Fusen allerhand Muschel...</td>\n",
+       "      <td>32cc068f-6a60-4040-aa4a-0575f48b5c52</td>\n",
+       "      <td>155.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>203</th>\n",
+       "      <td>('r. Sie kennen kein ander Spaziergehen, --- 0...</td>\n",
+       "      <td>638cd5aa-72b7-46a0-bbf1-82ec0f884747</td>\n",
+       "      <td>186.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>204</th>\n",
+       "      <td>('s. In dem Departement, wo ich wohne und in d...</td>\n",
+       "      <td>8438c431-385f-40f0-b167-51527c2c8e7e</td>\n",
+       "      <td>221.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>205</th>\n",
+       "      <td>('t. Ich wuste, das er besucht zu werden uerdi...</td>\n",
+       "      <td>41bd83f1-06ec-4186-af2e-63dea256dfc8</td>\n",
+       "      <td>267.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>206</th>\n",
+       "      <td>('n. Sie hatten in ihrem Hause ein Paar rothe ...</td>\n",
+       "      <td>e5ebef45-e36c-4742-9976-702dfad43fc5</td>\n",
+       "      <td>270.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>207</th>\n",
+       "      <td>('e. Nach diesem Kaufe nahmen wir uon dem gute...</td>\n",
+       "      <td>a02a872b-8602-4ff4-93bd-f539ff93ada6</td>\n",
+       "      <td>276.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>208</th>\n",
+       "      <td>('n. Der Viceconsul, rein Drogman und ein fran...</td>\n",
+       "      <td>a5b12bfb-3a78-46ac-8d62-73d5bf4425c9</td>\n",
+       "      <td>277.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>209</th>\n",
+       "      <td>('n. Unsere Esel Fielen nieder, sanken in den ...</td>\n",
+       "      <td>ceed7937-1824-4222-ba6c-686b2b4d7738</td>\n",
+       "      <td>278.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>210</th>\n",
+       "      <td>('tu Wir hatten Maulefel, womit man gewohnlich...</td>\n",
+       "      <td>94d76c3d-1115-4a4e-91b5-501f6e88de65</td>\n",
+       "      <td>386.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>211</th>\n",
+       "      <td>('. Ich trug kein Bedenken, mich ihnen anzuuer...</td>\n",
+       "      <td>acc36476-9315-4751-9646-bba3a2231e07</td>\n",
+       "      <td>386.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>212</th>\n",
+       "      <td>('e. Damit man nun nicht an meiner Furchtlosig...</td>\n",
+       "      <td>b9449bfd-c3bb-4b97-9c45-305b10b7aeca</td>\n",
+       "      <td>389.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>213</th>\n",
+       "      <td>('). Bei meiner Rukkunft nach Abukir fand ich ...</td>\n",
+       "      <td>995c9029-3950-44cb-a7c7-e6f91a96ecf7</td>\n",
+       "      <td>391.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>214</th>\n",
+       "      <td>('-- Plinius berichtet nach dem Xenophon, die ...</td>\n",
+       "      <td>fe020016-1d51-4a1f-bbc1-d4a65973bf67</td>\n",
+       "      <td>402.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>215</th>\n",
+       "      <td>('g. Ehe wir noch an dieseru Ore anlangten, sa...</td>\n",
+       "      <td>4795c52d-1f6e-4de6-b87f-a5ae17735dad</td>\n",
+       "      <td>419.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>216</th>\n",
+       "      <td>('ort. Wir kamen auf einen\\' bedekten Sand, de...</td>\n",
+       "      <td>92f4ee0e-d411-4c55-bda6-02b5f8aefce6</td>\n",
+       "      <td>433.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>217</th>\n",
+       "      <td>('n. Ob fie gleich in starkem Galop auf uns zu...</td>\n",
+       "      <td>0c529bb3-6ce7-4a2f-8741-1aaa4f2205d4</td>\n",
+       "      <td>434.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>218</th>\n",
+       "      <td>('li Wenn man den Arabern glaubt, die der Mein...</td>\n",
+       "      <td>9a71ef03-d718-402b-b7f9-14844f486a22</td>\n",
+       "      <td>452.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>219</th>\n",
+       "      <td>('t, Ich hatte einen uon den Landleuten, die i...</td>\n",
+       "      <td>e6e36f7b-3376-4624-a48b-ff2d88ba6167</td>\n",
+       "      <td>460.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>220</th>\n",
+       "      <td>('91 Ich uerlies endlich diesen. heltischer Au...</td>\n",
+       "      <td>39769078-50ac-4faf-b32a-e74fd7cc46b0</td>\n",
+       "      <td>462.0</td>\n",
+       "      <td>Z166069305_merged.txt</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                   sentence_annotation  \\\n",
+       "200  ('r. Wir giengen in geschloffenen Reihen und m...   \n",
+       "201  ('s. Da die Wagen nicht gewohnlich sind, so be...   \n",
+       "202  ('t. Man tritt mit den Fusen allerhand Muschel...   \n",
+       "203  ('r. Sie kennen kein ander Spaziergehen, --- 0...   \n",
+       "204  ('s. In dem Departement, wo ich wohne und in d...   \n",
+       "205  ('t. Ich wuste, das er besucht zu werden uerdi...   \n",
+       "206  ('n. Sie hatten in ihrem Hause ein Paar rothe ...   \n",
+       "207  ('e. Nach diesem Kaufe nahmen wir uon dem gute...   \n",
+       "208  ('n. Der Viceconsul, rein Drogman und ein fran...   \n",
+       "209  ('n. Unsere Esel Fielen nieder, sanken in den ...   \n",
+       "210  ('tu Wir hatten Maulefel, womit man gewohnlich...   \n",
+       "211  ('. Ich trug kein Bedenken, mich ihnen anzuuer...   \n",
+       "212  ('e. Damit man nun nicht an meiner Furchtlosig...   \n",
+       "213  ('). Bei meiner Rukkunft nach Abukir fand ich ...   \n",
+       "214  ('-- Plinius berichtet nach dem Xenophon, die ...   \n",
+       "215  ('g. Ehe wir noch an dieseru Ore anlangten, sa...   \n",
+       "216  ('ort. Wir kamen auf einen\\' bedekten Sand, de...   \n",
+       "217  ('n. Ob fie gleich in starkem Galop auf uns zu...   \n",
+       "218  ('li Wenn man den Arabern glaubt, die der Mein...   \n",
+       "219  ('t, Ich hatte einen uon den Landleuten, die i...   \n",
+       "220  ('91 Ich uerlies endlich diesen. heltischer Au...   \n",
+       "\n",
+       "                            id_annotation  page_annotation  \\\n",
+       "200  a406cecb-48dc-4b27-9e2e-e831301a59bc            149.0   \n",
+       "201  53e38871-8353-4ea1-acc5-e60964d4b02d            151.0   \n",
+       "202  32cc068f-6a60-4040-aa4a-0575f48b5c52            155.0   \n",
+       "203  638cd5aa-72b7-46a0-bbf1-82ec0f884747            186.0   \n",
+       "204  8438c431-385f-40f0-b167-51527c2c8e7e            221.0   \n",
+       "205  41bd83f1-06ec-4186-af2e-63dea256dfc8            267.0   \n",
+       "206  e5ebef45-e36c-4742-9976-702dfad43fc5            270.0   \n",
+       "207  a02a872b-8602-4ff4-93bd-f539ff93ada6            276.0   \n",
+       "208  a5b12bfb-3a78-46ac-8d62-73d5bf4425c9            277.0   \n",
+       "209  ceed7937-1824-4222-ba6c-686b2b4d7738            278.0   \n",
+       "210  94d76c3d-1115-4a4e-91b5-501f6e88de65            386.0   \n",
+       "211  acc36476-9315-4751-9646-bba3a2231e07            386.0   \n",
+       "212  b9449bfd-c3bb-4b97-9c45-305b10b7aeca            389.0   \n",
+       "213  995c9029-3950-44cb-a7c7-e6f91a96ecf7            391.0   \n",
+       "214  fe020016-1d51-4a1f-bbc1-d4a65973bf67            402.0   \n",
+       "215  4795c52d-1f6e-4de6-b87f-a5ae17735dad            419.0   \n",
+       "216  92f4ee0e-d411-4c55-bda6-02b5f8aefce6            433.0   \n",
+       "217  0c529bb3-6ce7-4a2f-8741-1aaa4f2205d4            434.0   \n",
+       "218  9a71ef03-d718-402b-b7f9-14844f486a22            452.0   \n",
+       "219  e6e36f7b-3376-4624-a48b-ff2d88ba6167            460.0   \n",
+       "220  39769078-50ac-4faf-b32a-e74fd7cc46b0            462.0   \n",
+       "\n",
+       "           file_annotation  \n",
+       "200  Z166069305_merged.txt  \n",
+       "201  Z166069305_merged.txt  \n",
+       "202  Z166069305_merged.txt  \n",
+       "203  Z166069305_merged.txt  \n",
+       "204  Z166069305_merged.txt  \n",
+       "205  Z166069305_merged.txt  \n",
+       "206  Z166069305_merged.txt  \n",
+       "207  Z166069305_merged.txt  \n",
+       "208  Z166069305_merged.txt  \n",
+       "209  Z166069305_merged.txt  \n",
+       "210  Z166069305_merged.txt  \n",
+       "211  Z166069305_merged.txt  \n",
+       "212  Z166069305_merged.txt  \n",
+       "213  Z166069305_merged.txt  \n",
+       "214  Z166069305_merged.txt  \n",
+       "215  Z166069305_merged.txt  \n",
+       "216  Z166069305_merged.txt  \n",
+       "217  Z166069305_merged.txt  \n",
+       "218  Z166069305_merged.txt  \n",
+       "219  Z166069305_merged.txt  \n",
+       "220  Z166069305_merged.txt  "
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Filter rows where 'found_in_text_vector' is False --> False Negatives\n",
+    "missed = matches_df[matches_df['found_in_text_vector'] == False]\n",
+    "missed_fil = missed['id_annotation'].drop_duplicates(keep='first')\n",
+    "missed_pages = missed['page_annotation'].drop_duplicates(keep='first')\n",
+    "\n",
+    "# Filter out pages that are in unique pages found\n",
+    "missed_pages_unique = missed_pages[~missed_pages.isin(unique_foundPages_list)]\n",
+    "retrieved_pages_otherVec = missed_pages[missed_pages.isin(unique_foundPages_list)]\n",
+    "\n",
+    "print(f\"False Negatives: Missed {len(missed_fil)} mentions on {len(missed_pages_unique)} pages, of which {len(retrieved_pages_otherVec)} pages were retrieved with another vector.\")\n",
+    "print(f\"({(len(missed_pages_unique))/len(unique_pages_list)*100:.2f}%).\")\n",
+    "\n",
+    "missed[['sentence_annotation', 'id_annotation', 'page_annotation', 'file_annotation']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "e7753ea4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Retrieved 173 vectors on 173 pages of which 11 pages do contain annotations (correct pages, but other vector retrieved).\n",
+      "and of which 162 pages do not contain annotations (= False Positives)\n",
+      "(405.00)\n",
+      "and of which 0 pages were retrieved with other vectors (= True Negatives).\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text_vector_x</th>\n",
+       "      <th>rerank</th>\n",
+       "      <th>id_text_vector</th>\n",
+       "      <th>page</th>\n",
+       "      <th>page_annotation</th>\n",
+       "      <th>file_annotation</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>Unter einer Regierung, die nur die Absicht bat...</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>9ee8949a-e0cc-4334-bc36-b71becd2d5c7</td>\n",
+       "      <td>400.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>feiner Neise 1. th.).</td>\n",
+       "      <td>24.0</td>\n",
+       "      <td>39e2e422-b7c3-40c1-90ee-12718c603261</td>\n",
+       "      <td>393.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>**) Ursprung der Gefese 2. Bd.</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>4ff5ac89-e2f3-468f-bad6-712ecce32e54</td>\n",
+       "      <td>170.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>Von frih morgens an, bis auf den Abend, hat ma...</td>\n",
+       "      <td>29.0</td>\n",
+       "      <td>d4357d62-9022-4b84-803b-e634b791a923</td>\n",
+       "      <td>182.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>Das Wasser, das * Strix passerina, Lin, Chevec...</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>7aaae2e6-1e90-46a2-9f05-fe28c7b2842b</td>\n",
+       "      <td>236.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>195</th>\n",
+       "      <td>fchichte uberhaupt, und jene des. Mungo insbes...</td>\n",
+       "      <td>196.0</td>\n",
+       "      <td>f8995776-d337-413e-bc6b-d54a1fb5ff86</td>\n",
+       "      <td>223.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>196</th>\n",
+       "      <td>Ob sie gleich gesehen hatten, was seit Morgens...</td>\n",
+       "      <td>197.0</td>\n",
+       "      <td>3315a574-24e7-4a73-b660-bd151a27956b</td>\n",
+       "      <td>437.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>197</th>\n",
+       "      <td>Die fehr oftern Uncinigkeiten zwischen den Beh...</td>\n",
+       "      <td>198.0</td>\n",
+       "      <td>bd45fa07-bd6a-46f3-9ba9-516d48d55fb5</td>\n",
+       "      <td>161.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>198</th>\n",
+       "      <td>Es ist bemerkenswerth, das man diesen unruhige...</td>\n",
+       "      <td>199.0</td>\n",
+       "      <td>736f2f9e-85b2-470d-a94e-f34f64db4189</td>\n",
+       "      <td>89.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>199</th>\n",
+       "      <td>M. 24 und 26. Wit find Flein auf Erden und Flu...</td>\n",
+       "      <td>200.0</td>\n",
+       "      <td>ebb43751-cbfe-40d5-9056-95e68b9102a6</td>\n",
+       "      <td>126.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>173 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                         text_vector_x  rerank  \\\n",
+       "16   Unter einer Regierung, die nur die Absicht bat...    17.0   \n",
+       "23                               feiner Neise 1. th.).    24.0   \n",
+       "27                      **) Ursprung der Gefese 2. Bd.    28.0   \n",
+       "28   Von frih morgens an, bis auf den Abend, hat ma...    29.0   \n",
+       "29   Das Wasser, das * Strix passerina, Lin, Chevec...    30.0   \n",
+       "..                                                 ...     ...   \n",
+       "195  fchichte uberhaupt, und jene des. Mungo insbes...   196.0   \n",
+       "196  Ob sie gleich gesehen hatten, was seit Morgens...   197.0   \n",
+       "197  Die fehr oftern Uncinigkeiten zwischen den Beh...   198.0   \n",
+       "198  Es ist bemerkenswerth, das man diesen unruhige...   199.0   \n",
+       "199  M. 24 und 26. Wit find Flein auf Erden und Flu...   200.0   \n",
+       "\n",
+       "                           id_text_vector   page  page_annotation  \\\n",
+       "16   9ee8949a-e0cc-4334-bc36-b71becd2d5c7  400.0              NaN   \n",
+       "23   39e2e422-b7c3-40c1-90ee-12718c603261  393.0              NaN   \n",
+       "27   4ff5ac89-e2f3-468f-bad6-712ecce32e54  170.0              NaN   \n",
+       "28   d4357d62-9022-4b84-803b-e634b791a923  182.0              NaN   \n",
+       "29   7aaae2e6-1e90-46a2-9f05-fe28c7b2842b  236.0              NaN   \n",
+       "..                                    ...    ...              ...   \n",
+       "195  f8995776-d337-413e-bc6b-d54a1fb5ff86  223.0              NaN   \n",
+       "196  3315a574-24e7-4a73-b660-bd151a27956b  437.0              NaN   \n",
+       "197  bd45fa07-bd6a-46f3-9ba9-516d48d55fb5  161.0              NaN   \n",
+       "198  736f2f9e-85b2-470d-a94e-f34f64db4189   89.0              NaN   \n",
+       "199  ebb43751-cbfe-40d5-9056-95e68b9102a6  126.0              NaN   \n",
+       "\n",
+       "    file_annotation  \n",
+       "16              NaN  \n",
+       "23              NaN  \n",
+       "27              NaN  \n",
+       "28              NaN  \n",
+       "29              NaN  \n",
+       "..              ...  \n",
+       "195             NaN  \n",
+       "196             NaN  \n",
+       "197             NaN  \n",
+       "198             NaN  \n",
+       "199             NaN  \n",
+       "\n",
+       "[173 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Filter rows where 'found_in_text_vector' is NaN --> False Positives\n",
+    "FP = matches_df[matches_df['found_in_text_vector'].isna()]\n",
+    "fP_pages = FP['page'].drop_duplicates(keep='first')\n",
+    "\n",
+    "# Extract the \"page\" value from each dictionary of unique pages found\n",
+    "unique_FPPages_list = fP_pages.to_list()\n",
+    "\n",
+    "# Filter out pages that are in unique pages found\n",
+    "ret_pages_Annot = fP_pages[fP_pages.isin(unique_pages_list)]\n",
+    "ret_pages_noAnnot = fP_pages[~fP_pages.isin(unique_pages_list)]\n",
+    "retrieved_otherVec = fP_pages[fP_pages.isin(unique_foundPages_list)]\n",
+    "\n",
+    "print(f\"Retrieved {len(FP)} vectors on {len(unique_FPPages_list)} pages of which {len(ret_pages_Annot)} pages do contain annotations (correct pages, but other vector retrieved).\")\n",
+    "print(f\"and of which {len(ret_pages_noAnnot)} pages do not contain annotations (= False Positives)\")\n",
+    "print(f\"({len(ret_pages_noAnnot)/len(unique_pages_list)*100:.2f})\")\n",
+    "print(f\"and of which {len(retrieved_otherVec)} pages were retrieved with other vectors (= True Negatives).\")\n",
+    "FP[['text_vector_x', 'rerank', 'id_text_vector', 'page', 'page_annotation', 'file_annotation']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2bf572ee",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/analysis/query_index.ipynb

@@ -0,0 +1,1685 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "73ef8baa",
+   "metadata": {},
+   "source": [
+    "# Query Marqo Index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "dee4e8d3",
+   "metadata": {
+    "code_folding": []
+   },
+   "outputs": [],
+   "source": [
+    "## Import packages\n",
+    "import marqo as mq\n",
+    "import pandas as pd\n",
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.display import Javascript\n",
+    "from pprint import pprint"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "e80297f9",
+   "metadata": {
+    "code_folding": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Connected to onit-sonnini-DHd2025-prep.\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Connect to Marqo\n",
+    "\n",
+    "MARQO_URL = \"http://your.ip:port\"\n",
+    "marqoClient = mq.Client(url=MARQO_URL)\n",
+    "#pprint(marqoClient.get_indexes())\n",
+    "\n",
+    "## DHd 2025 ##\n",
+    "indexName = \"onit-sonnini-DHd2025-prep\" ## index with LLM-corrected texts as tensor field\n",
+    "#indexName = \"onit-sonnini-DHd2025-clean\" ## index with cleaned texts as tensor field\n",
+    "print(f'Connected to {indexName}.')\n",
+    "\n",
+    "# Load corpus data\n",
+    "bc_corpus = pd.read_csv(\"data/ONiT_barcodes_ALL_metadata_ONB_status_2024-05-23.csv\")\n",
+    "bc_corpus = bc_corpus.drop_duplicates(subset='barcode', keep='last')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "353081b8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'message': 'Welcome to Marqo', 'version': '2.5.1'}"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "marqoClient.index(indexName).get_marqo()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c9ceb6de",
+   "metadata": {
+    "code_folding": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>barcode</th>\n",
+       "      <th>page</th>\n",
+       "      <th>iiif_link</th>\n",
+       "      <th>text_clean</th>\n",
+       "      <th>text_orig</th>\n",
+       "      <th>text_prep</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>5</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>C. S.' Sonnin is,\\nehemaligen Dffiziers uub In...</td>\n",
+       "      <td>!\\n\\nC. S.' Sonnin i’s,\\n\\n;\\nehemaligen Dffiz...</td>\n",
+       "      <td>C. S.' Sonnini's,\\nehemaligen Offiziers und In...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>6</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>2125 murid\\ngobiothers\\nconale\\nKOENISE\\nKAISE...</td>\n",
+       "      <td>2125 murid\\n\\ngobiothers\\n\\nconale\\n\\nܪܝ\\n\\n، ...</td>\n",
+       "      <td>2255 Murdoch \\nGouverneurs\\nComte\\nKOENIGSE\\nK...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>7</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>V o r re o e\\nDe 6 u i berpe Bet,$.\\nundteichl...</td>\n",
+       "      <td>V o r re o e\\nDė 6 u i berpe Bét,$.\\n\\nundteic...</td>\n",
+       "      <td>Vorrede.\\n\\nDeutschland hat in betreff seiner ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>8</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>Welt auf fich gezogen haben. Viele feiner Denk...</td>\n",
+       "      <td>IV\\n\\n.\\n\\nWelt auf fich gezogen haben. Viele ...</td>\n",
+       "      <td>Welt auf sich gezogen haben. Viele seiner Denk...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>9</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>und auslandische Sklaven genossen die Vortheil...</td>\n",
+       "      <td>)\\n\\n-\\nI\\n\\nI\\n\\n1\\n\\n11\\n\\nr.\\n\\nund ausländ...</td>\n",
+       "      <td>Und ausländische Sklaven genossen die Vortheil...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>486</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>494</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>&lt;empty page&gt;</td>\n",
+       "      <td>+\\n\\nè³½\\n\\n1\\n\\n\\n</td>\n",
+       "      <td>&lt;empty page&gt;</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>487</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>495</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>Riedel fo\\nBedminenzelt</td>\n",
+       "      <td>.7.\\n\\n.3.6\\n\\n&gt;\\n\\n1 0\\n\\nRiedel fo\\n\\n(\\n\\n(...</td>\n",
+       "      <td>Riedel von Eisenbach</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>488</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>499</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>Osterreichische Nationalbibliothek\\n+ Z166069305</td>\n",
+       "      <td>Österreichische Nationalbibliothek\\n\\n+ Z1660...</td>\n",
+       "      <td>Please provide the faulty OCR texts generated ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>489</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>503</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>&lt;empty page&gt;</td>\n",
+       "      <td>{\\n  \"status code\" : 404,\\n  \"message\" : \"The ...</td>\n",
+       "      <td>&lt;empty page&gt;</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>490</th>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>504</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>&lt;empty page&gt;</td>\n",
+       "      <td>{\\n  \"status code\" : 404,\\n  \"message\" : \"The ...</td>\n",
+       "      <td>&lt;empty page&gt;</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>491 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        barcode  page                                          iiif_link  \\\n",
+       "0    Z166069305     5  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "1    Z166069305     6  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "2    Z166069305     7  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "3    Z166069305     8  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "4    Z166069305     9  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "..          ...   ...                                                ...   \n",
+       "486  Z166069305   494  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "487  Z166069305   495  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "488  Z166069305   499  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "489  Z166069305   503  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "490  Z166069305   504  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "\n",
+       "                                            text_clean  \\\n",
+       "0    C. S.' Sonnin is,\\nehemaligen Dffiziers uub In...   \n",
+       "1    2125 murid\\ngobiothers\\nconale\\nKOENISE\\nKAISE...   \n",
+       "2    V o r re o e\\nDe 6 u i berpe Bet,$.\\nundteichl...   \n",
+       "3    Welt auf fich gezogen haben. Viele feiner Denk...   \n",
+       "4    und auslandische Sklaven genossen die Vortheil...   \n",
+       "..                                                 ...   \n",
+       "486                                       <empty page>   \n",
+       "487                            Riedel fo\\nBedminenzelt   \n",
+       "488   Osterreichische Nationalbibliothek\\n+ Z166069305   \n",
+       "489                                       <empty page>   \n",
+       "490                                       <empty page>   \n",
+       "\n",
+       "                                             text_orig  \\\n",
+       "0    !\\n\\nC. S.' Sonnin i’s,\\n\\n;\\nehemaligen Dffiz...   \n",
+       "1    2125 murid\\n\\ngobiothers\\n\\nconale\\n\\nܪܝ\\n\\n، ...   \n",
+       "2    V o r re o e\\nDė 6 u i berpe Bét,$.\\n\\nundteic...   \n",
+       "3    IV\\n\\n.\\n\\nWelt auf fich gezogen haben. Viele ...   \n",
+       "4    )\\n\\n-\\nI\\n\\nI\\n\\n1\\n\\n11\\n\\nr.\\n\\nund ausländ...   \n",
+       "..                                                 ...   \n",
+       "486                                +\\n\\nè³½\\n\\n1\\n\\n\\n   \n",
+       "487  .7.\\n\\n.3.6\\n\\n>\\n\\n1 0\\n\\nRiedel fo\\n\\n(\\n\\n(...   \n",
+       "488  Österreichische Nationalbibliothek\\n\\n+ Z1660...   \n",
+       "489  {\\n  \"status code\" : 404,\\n  \"message\" : \"The ...   \n",
+       "490  {\\n  \"status code\" : 404,\\n  \"message\" : \"The ...   \n",
+       "\n",
+       "                                             text_prep  \n",
+       "0    C. S.' Sonnini's,\\nehemaligen Offiziers und In...  \n",
+       "1    2255 Murdoch \\nGouverneurs\\nComte\\nKOENIGSE\\nK...  \n",
+       "2    Vorrede.\\n\\nDeutschland hat in betreff seiner ...  \n",
+       "3    Welt auf sich gezogen haben. Viele seiner Denk...  \n",
+       "4    Und ausländische Sklaven genossen die Vortheil...  \n",
+       "..                                                 ...  \n",
+       "486                                       <empty page>  \n",
+       "487                               Riedel von Eisenbach  \n",
+       "488  Please provide the faulty OCR texts generated ...  \n",
+       "489                                       <empty page>  \n",
+       "490                                       <empty page>  \n",
+       "\n",
+       "[491 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Preview data\n",
+    "import pandas as pd\n",
+    "\n",
+    "index_DHd = pd.read_csv(\"C:/onit_rag/data/DHd_index-cleaned.csv\")\n",
+    "index_DHd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "203aa5ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Query parameter\n",
+    "\n",
+    "query = \"Pferd, Pferde\"\n",
+    "limit = 1000 # max limit = 1000\n",
+    "threshold = 0.7 # set threshold\n",
+    "filter_string = \"barcode:(Z166069305)\" # Sonnini Bd. 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "693ec044",
+   "metadata": {
+    "code_folding": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Output tensor search:  491\n",
+      "Output lexical search:  28\n",
+      "Output tensor search after filtering vectors < 5 tokens:  447\n",
+      "Output tensor search with threshold:  447\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Query the index\n",
+    "\n",
+    "#results = marqoClient.index(indexName).search(q=query, limit=limit) # basic search\n",
+    "results_tensor = marqoClient.index(indexName).search(q=query, limit=limit, filter_string=filter_string) # tensor search\n",
+    "print('Output tensor search: ', len(results_tensor['hits']))\n",
+    "results_lexical = marqoClient.index(indexName).search(q=query, limit=limit, filter_string=filter_string, search_method=\"LEXICAL\") # keyword search (BM25)\n",
+    "print('Output lexical search: ', len(results_lexical['hits']))\n",
+    "\n",
+    "## Load into dataframes\n",
+    "\n",
+    "# results tensor search\n",
+    "output1 = pd.DataFrame(results_tensor[\"hits\"])\n",
+    "# Filter rows where _highlights are 5 tokens or more\n",
+    "output1 = output1[output1[\"_highlights\"].apply(lambda x: len(str(x).split()) >= 5)]\n",
+    "print('Output tensor search after filtering vectors < 5 tokens: ', len(output1))\n",
+    "### Filter rows where _score is >= threshold\n",
+    "output1 = output1[output1[\"_score\"] >= threshold]\n",
+    "print('Output tensor search with threshold: ', len(output1))\n",
+    "# Add corpus metadata\n",
+    "# Merge the two DataFrames on the 'barcode' column to add 'corpus'\n",
+    "output1 = output1.merge(bc_corpus, on='barcode', how='inner')\n",
+    "# Add index +1 as a new column called 'rank'\n",
+    "output1['rank'] = output1.index + 1\n",
+    "# Add document identifier\n",
+    "output1['document'] = output1.apply(lambda row: f\"{row['barcode']}_{row['page']}\", axis=1)\n",
+    "\n",
+    "# results keyword search\n",
+    "if len(results_lexical['hits']) == 0:\n",
+    "    output2 = output1\n",
+    "    print(\"No lexical search results! Using only tensor results.\")\n",
+    "    \n",
+    "else:\n",
+    "    output2 = pd.DataFrame(results_lexical[\"hits\"])\n",
+    "    # Add corpus metadata\n",
+    "    # Merge the two DataFrames on the 'barcode' column to add 'corpus'\n",
+    "    output2 = output2.merge(bc_corpus, on='barcode', how='inner')\n",
+    "    # Add index +1 as a new column called 'rank'\n",
+    "    output2['rank'] = output2.index + 1#apply(lambda row: output2.index + 1, axis=1)\n",
+    "    # Add document identifier\n",
+    "    output2['document'] = output2.apply(lambda row: f\"{row['barcode']}_{row['page']}\", axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "545badce",
+   "metadata": {
+    "code_folding": [
+     0
+    ]
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>document</th>\n",
+       "      <th>rrf_score</th>\n",
+       "      <th>barcode</th>\n",
+       "      <th>page</th>\n",
+       "      <th>iiif_link</th>\n",
+       "      <th>text_orig</th>\n",
+       "      <th>text_clean</th>\n",
+       "      <th>text_prep</th>\n",
+       "      <th>_id</th>\n",
+       "      <th>_highlights</th>\n",
+       "      <th>_score</th>\n",
+       "      <th>rank</th>\n",
+       "      <th>corpus</th>\n",
+       "      <th>rerank</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Z166069305_430</td>\n",
+       "      <td>0.032522</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>430</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>402  befand mich in einiger Entfernung davon, ...</td>\n",
+       "      <td>befand mich in einiger Entfernung davon, und d...</td>\n",
+       "      <td>befand mich in einiger Entfernung davon, und d...</td>\n",
+       "      <td>7e2b21a1-4cdb-4b6a-b1c0-75c30bfe13bb</td>\n",
+       "      <td>[{'text_prep': 'befand mich in einiger Entfern...</td>\n",
+       "      <td>0.892140</td>\n",
+       "      <td>1/2</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Z166069305_10</td>\n",
+       "      <td>0.030118</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>10</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>VI  große Flinten, die ihm ſeine Diener zur Se...</td>\n",
+       "      <td>grose Flinten, die ihm seine Diener zur Seite ...</td>\n",
+       "      <td>große Flinten, die ihm seine Diener zur Seite ...</td>\n",
+       "      <td>ab0905d4-6ca7-4e94-8fb2-3d2081632d6d</td>\n",
+       "      <td>[{'text_prep': 'Sein letztes Hulfritt trifft z...</td>\n",
+       "      <td>0.872264</td>\n",
+       "      <td>4/9</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Z166069305_399</td>\n",
+       "      <td>0.029324</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>399</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>1  1  1  ,  571 Plinius berichtet nach dem Xen...</td>\n",
+       "      <td>Plinius berichtet nach dem Xenophon, die Camel...</td>\n",
+       "      <td>Plinius berichtet nach dem Xenophon, die Camel...</td>\n",
+       "      <td>feb1b47b-effe-4e9d-be0f-7a749fed5ec0</td>\n",
+       "      <td>[{'text_prep': 'Ich muss hier auch bemerken, d...</td>\n",
+       "      <td>0.865306</td>\n",
+       "      <td>13/4</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Z166069305_415</td>\n",
+       "      <td>0.028860</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>415</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>587  nen. Ich ritt auf fie los, aber Hufrein l...</td>\n",
+       "      <td>nen. Ich ritt auf fie los, aber Hufrein lies m...</td>\n",
+       "      <td>Ich ritt auf sie los, aber Hufrein lies mich r...</td>\n",
+       "      <td>02bb750e-0993-4341-ab74-934d28fc523e</td>\n",
+       "      <td>[{'text_prep': 'Ehe wir noch an dieser Ore anl...</td>\n",
+       "      <td>0.864078</td>\n",
+       "      <td>17/3</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Z166069305_220</td>\n",
+       "      <td>0.028485</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>220</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>' 92  lungen und Bewegungen. Mit einer Phyſiog...</td>\n",
+       "      <td>lungen und Bewegungen. Mit einer Physiognomie,...</td>\n",
+       "      <td>Lungen und Bewegungen. Mit einer Physiognomie,...</td>\n",
+       "      <td>7949fe4e-5534-4522-b151-35b57a733650</td>\n",
+       "      <td>[{'text_prep': 'Die Haustiere finden nirgends ...</td>\n",
+       "      <td>0.869331</td>\n",
+       "      <td>6/15</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>5</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>478</th>\n",
+       "      <td>Z166069305_493</td>\n",
+       "      <td>0.001901</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>493</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>Pl. 5.   Fig.2.  Fig.3.  Fig. 1  B  Riedel fee...</td>\n",
+       "      <td>Pl. 5. Fig.2. Fig.3. Fig. 1 Riedel fee. Fig. 1...</td>\n",
+       "      <td>Pl. 5. Fig. 2. Fig. 3. Fig. 1 Riedel fee. Fig....</td>\n",
+       "      <td>131e6f12-db13-422f-96e4-0924fe19026b</td>\n",
+       "      <td>[{'text_prep': 'Fig. 1 Riedel fee.'}]</td>\n",
+       "      <td>0.819874</td>\n",
+       "      <td>466</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>479</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>479</th>\n",
+       "      <td>Z166069305_484</td>\n",
+       "      <td>0.001898</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>484</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>1  -  1  Seite 109 Zeile I v. u. Abanſoon lies...</td>\n",
+       "      <td>Seite 109 Zeile I v. u. Abansoon lies as anson...</td>\n",
+       "      <td>Seite 109 Zeile I v. u. Abends sonnig, ansonst...</td>\n",
+       "      <td>da3ced63-91a8-4710-81b6-40da8a1cf407</td>\n",
+       "      <td>[{'text_prep': 'Seite 109 Zeile I v. u. Abends...</td>\n",
+       "      <td>0.818248</td>\n",
+       "      <td>467</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>480</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>480</th>\n",
+       "      <td>Z166069305_489</td>\n",
+       "      <td>0.001894</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>489</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>Th.  PL 3  ஏப்ரதம் THUNDE  1  4  Ironis  ту  C...</td>\n",
+       "      <td>Th. PL 3 THUNDE Ironis Cine kanclerte tatue be...</td>\n",
+       "      <td>Theodor Pl. 3 Thundersturm  Ironische cine kan...</td>\n",
+       "      <td>228e98a9-2d29-4f34-a0c2-adcdd2eeff09</td>\n",
+       "      <td>[{'text_prep': 'Theodor Pl. 3 Thundersturm  Ir...</td>\n",
+       "      <td>0.818178</td>\n",
+       "      <td>468</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>481</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>481</th>\n",
+       "      <td>Z166069305_482</td>\n",
+       "      <td>0.001890</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>482</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>ie  \" !!!  3  iC; 1: 0 Wis . bici\" ..)  .\"., 6...</td>\n",
+       "      <td>iC; 1: 0 Wis . bici\" ..) .\"., 6,7, ART 9 * noi...</td>\n",
+       "      <td>Ich kam also um die neunte Stunde   zu Wismar.</td>\n",
+       "      <td>9e9b9254-c675-4427-a20e-88b8d723aec3</td>\n",
+       "      <td>[{'text_prep': 'Ich kam also um die neunte Stu...</td>\n",
+       "      <td>0.812620</td>\n",
+       "      <td>469</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>482</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>482</th>\n",
+       "      <td>Z166069305_499</td>\n",
+       "      <td>0.001887</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>499</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>Österreichische Nationalbibliothek  + Z166069305</td>\n",
+       "      <td>Osterreichische Nationalbibliothek + Z166069305</td>\n",
+       "      <td>Please provide the faulty OCR texts generated ...</td>\n",
+       "      <td>93723fd2-f359-4251-8621-513c4f6e1128</td>\n",
+       "      <td>[{'text_prep': '(Please paste the text)'}]</td>\n",
+       "      <td>0.805837</td>\n",
+       "      <td>470</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>483</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>483 rows × 14 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           document  rrf_score     barcode page  \\\n",
+       "0    Z166069305_430   0.032522  Z166069305  430   \n",
+       "1     Z166069305_10   0.030118  Z166069305   10   \n",
+       "2    Z166069305_399   0.029324  Z166069305  399   \n",
+       "3    Z166069305_415   0.028860  Z166069305  415   \n",
+       "4    Z166069305_220   0.028485  Z166069305  220   \n",
+       "..              ...        ...         ...  ...   \n",
+       "478  Z166069305_493   0.001901  Z166069305  493   \n",
+       "479  Z166069305_484   0.001898  Z166069305  484   \n",
+       "480  Z166069305_489   0.001894  Z166069305  489   \n",
+       "481  Z166069305_482   0.001890  Z166069305  482   \n",
+       "482  Z166069305_499   0.001887  Z166069305  499   \n",
+       "\n",
+       "                                             iiif_link  \\\n",
+       "0    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "1    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "2    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "3    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "4    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "..                                                 ...   \n",
+       "478  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "479  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "480  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "481  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "482  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "\n",
+       "                                             text_orig  \\\n",
+       "0    402  befand mich in einiger Entfernung davon, ...   \n",
+       "1    VI  große Flinten, die ihm ſeine Diener zur Se...   \n",
+       "2    1  1  1  ,  571 Plinius berichtet nach dem Xen...   \n",
+       "3    587  nen. Ich ritt auf fie los, aber Hufrein l...   \n",
+       "4    ' 92  lungen und Bewegungen. Mit einer Phyſiog...   \n",
+       "..                                                 ...   \n",
+       "478  Pl. 5.   Fig.2.  Fig.3.  Fig. 1  B  Riedel fee...   \n",
+       "479  1  -  1  Seite 109 Zeile I v. u. Abanſoon lies...   \n",
+       "480  Th.  PL 3  ஏப்ரதம் THUNDE  1  4  Ironis  ту  C...   \n",
+       "481  ie  \" !!!  3  iC; 1: 0 Wis . bici\" ..)  .\"., 6...   \n",
+       "482  Österreichische Nationalbibliothek  + Z166069305   \n",
+       "\n",
+       "                                            text_clean  \\\n",
+       "0    befand mich in einiger Entfernung davon, und d...   \n",
+       "1    grose Flinten, die ihm seine Diener zur Seite ...   \n",
+       "2    Plinius berichtet nach dem Xenophon, die Camel...   \n",
+       "3    nen. Ich ritt auf fie los, aber Hufrein lies m...   \n",
+       "4    lungen und Bewegungen. Mit einer Physiognomie,...   \n",
+       "..                                                 ...   \n",
+       "478  Pl. 5. Fig.2. Fig.3. Fig. 1 Riedel fee. Fig. 1...   \n",
+       "479  Seite 109 Zeile I v. u. Abansoon lies as anson...   \n",
+       "480  Th. PL 3 THUNDE Ironis Cine kanclerte tatue be...   \n",
+       "481  iC; 1: 0 Wis . bici\" ..) .\"., 6,7, ART 9 * noi...   \n",
+       "482    Osterreichische Nationalbibliothek + Z166069305   \n",
+       "\n",
+       "                                             text_prep  \\\n",
+       "0    befand mich in einiger Entfernung davon, und d...   \n",
+       "1    große Flinten, die ihm seine Diener zur Seite ...   \n",
+       "2    Plinius berichtet nach dem Xenophon, die Camel...   \n",
+       "3    Ich ritt auf sie los, aber Hufrein lies mich r...   \n",
+       "4    Lungen und Bewegungen. Mit einer Physiognomie,...   \n",
+       "..                                                 ...   \n",
+       "478  Pl. 5. Fig. 2. Fig. 3. Fig. 1 Riedel fee. Fig....   \n",
+       "479  Seite 109 Zeile I v. u. Abends sonnig, ansonst...   \n",
+       "480  Theodor Pl. 3 Thundersturm  Ironische cine kan...   \n",
+       "481     Ich kam also um die neunte Stunde   zu Wismar.   \n",
+       "482  Please provide the faulty OCR texts generated ...   \n",
+       "\n",
+       "                                      _id  \\\n",
+       "0    7e2b21a1-4cdb-4b6a-b1c0-75c30bfe13bb   \n",
+       "1    ab0905d4-6ca7-4e94-8fb2-3d2081632d6d   \n",
+       "2    feb1b47b-effe-4e9d-be0f-7a749fed5ec0   \n",
+       "3    02bb750e-0993-4341-ab74-934d28fc523e   \n",
+       "4    7949fe4e-5534-4522-b151-35b57a733650   \n",
+       "..                                    ...   \n",
+       "478  131e6f12-db13-422f-96e4-0924fe19026b   \n",
+       "479  da3ced63-91a8-4710-81b6-40da8a1cf407   \n",
+       "480  228e98a9-2d29-4f34-a0c2-adcdd2eeff09   \n",
+       "481  9e9b9254-c675-4427-a20e-88b8d723aec3   \n",
+       "482  93723fd2-f359-4251-8621-513c4f6e1128   \n",
+       "\n",
+       "                                           _highlights    _score  rank corpus  \\\n",
+       "0    [{'text_prep': 'befand mich in einiger Entfern...  0.892140   1/2    D19   \n",
+       "1    [{'text_prep': 'Sein letztes Hulfritt trifft z...  0.872264   4/9    D19   \n",
+       "2    [{'text_prep': 'Ich muss hier auch bemerken, d...  0.865306  13/4    D19   \n",
+       "3    [{'text_prep': 'Ehe wir noch an dieser Ore anl...  0.864078  17/3    D19   \n",
+       "4    [{'text_prep': 'Die Haustiere finden nirgends ...  0.869331  6/15    D19   \n",
+       "..                                                 ...       ...   ...    ...   \n",
+       "478              [{'text_prep': 'Fig. 1 Riedel fee.'}]  0.819874   466    D19   \n",
+       "479  [{'text_prep': 'Seite 109 Zeile I v. u. Abends...  0.818248   467    D19   \n",
+       "480  [{'text_prep': 'Theodor Pl. 3 Thundersturm  Ir...  0.818178   468    D19   \n",
+       "481  [{'text_prep': 'Ich kam also um die neunte Stu...  0.812620   469    D19   \n",
+       "482         [{'text_prep': '(Please paste the text)'}]  0.805837   470    D19   \n",
+       "\n",
+       "     rerank  \n",
+       "0         1  \n",
+       "1         2  \n",
+       "2         3  \n",
+       "3         4  \n",
+       "4         5  \n",
+       "..      ...  \n",
+       "478     479  \n",
+       "479     480  \n",
+       "480     481  \n",
+       "481     482  \n",
+       "482     483  \n",
+       "\n",
+       "[483 rows x 14 columns]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "## Reciprocal Rank Fusion (RRF)\n",
+    "\n",
+    "# Set the RRF parameter\n",
+    "k = 60\n",
+    "\n",
+    "# Concatenate the DataFrames to combine all rankings into one DataFrame\n",
+    "combined_df = pd.concat([output1, output2], ignore_index=True)\n",
+    "\n",
+    "# Initialize the RRF score column to 0\n",
+    "combined_df['rrf_score'] = 0\n",
+    "\n",
+    "# Function to update RRF scores\n",
+    "def update_rrf_scores(df, k):\n",
+    "    # Calculate RRF score using the formula and add to the rrf_score column\n",
+    "    df['rrf_score'] += 1 / (k + df['rank'])\n",
+    "\n",
+    "# Update RRF scores for the combined DataFrame\n",
+    "update_rrf_scores(combined_df, k)\n",
+    "\n",
+    "# Custom aggregation function to combine values with a '/'\n",
+    "def combine_values(values):\n",
+    "    # Convert to string and join unique values with a '/' separator\n",
+    "    return '/'.join(values.astype(str).unique())\n",
+    "\n",
+    "# Group by 'item' and sum the RRF scores for each item\n",
+    "final_scores_df = final_scores_df = combined_df.groupby('document', as_index=False).agg({\n",
+    "    'rrf_score': 'sum',\n",
+    "    'barcode': 'first',\n",
+    "    'page': 'first',\n",
+    "    'iiif_link': 'first',\n",
+    "    'text_orig': 'first',\n",
+    "    'text_clean': 'first',\n",
+    "    'text_prep': 'first',\n",
+    "    '_id': 'first',\n",
+    "    '_highlights': 'first',\n",
+    "    '_score': 'first',\n",
+    "    'rank': combine_values,\n",
+    "    'corpus': 'first'\n",
+    "})\n",
+    "\n",
+    "# Sort by total RRF score in descending order and reset index\n",
+    "final_scores_df = final_scores_df.sort_values(by='rrf_score', ascending=False).reset_index(drop=True)\n",
+    "\n",
+    "# Add rerank based on the sorted order\n",
+    "final_scores_df['rerank'] = final_scores_df.index + 1\n",
+    "\n",
+    "# Display the final DataFrame\n",
+    "#print(f'Total results: {len(final_scores_df)}')\n",
+    "#final_scores_df\n",
+    "\n",
+    "final_scores_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47aad8d3",
+   "metadata": {
+    "code_folding": [
+     0
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "## Multi-Term Queries\n",
+    "\n",
+    "query = 'Nur Pflanzen, Vegetation und Flora'\n",
+    "limit = 1000 # max limit = 1000\n",
+    "threshold = 0.80 # set threshold\n",
+    "filter_string = \"barcode:(Z166069305) OR barcode:(Z166069408)\" # Sonnini\n",
+    "\n",
+    "results = marqoClient.index(indexName).search(\n",
+    "    {\n",
+    "        query: 2.0,\n",
+    "        \"Tiere oder jegliche Fauna\": -0.8,\n",
+    "        \"wichtige Ernährung für die Bevölkerung\": +0.8,\n",
+    "        #\"Meteorologische Beobachtungen\": -0.8,\n",
+    "    },\n",
+    "    limit=limit,\n",
+    "    #filter_string=filter_string\n",
+    ")\n",
+    "\n",
+    "output = pd.DataFrame(results[\"hits\"])\n",
+    "output = output[output['_score'] > threshold] # filter results above threshold\n",
+    "\n",
+    "print(f'Total results: {len(output)}')\n",
+    "#output"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7cd84368",
+   "metadata": {},
+   "source": [
+    "# Generate Preview"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a045bdfd",
+   "metadata": {
+    "code_folding": [
+     0
+    ],
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "52fc4bb25414483dad663802648a3559",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(Box(children=(HBox(children=(HTML(value='<b>1</b>'), HTML(value='<i>Z166069305_430: </i><mark>b…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "334de0628ea440049482be20848ffcf7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output(layout=Layout(border='1px solid black', height='auto', padding='10px', width='100%'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "## Preview R matches\n",
+    "\n",
+    "R=20 # set number of results to be previewed\n",
+    "\n",
+    "# Function to unpack and concatenate all texts from the lists of dictionaries\n",
+    "def unpack_texts(series):\n",
+    "    return series.apply(lambda x: ' '.join([d.get('text_clean', d.get('text_prep', '')) for d in x if isinstance(d, dict)]))\n",
+    "\n",
+    "# Apply the function to the '_highlights' column and create a new column 'unpacked_highlights'\n",
+    "final_scores_df['unpacked_highlights'] = unpack_texts(final_scores_df['_highlights'])\n",
+    "#print(output['_highlights'])\n",
+    "\n",
+    "text_col = 'text_prep' if 'text_prep' in final_scores_df.columns else 'text_clean'\n",
+    "\n",
+    "# Check if 'unpacked_highlights' is empty and replace it if necessary\n",
+    "final_scores_df['unpacked_highlights'] = final_scores_df.apply(\n",
+    "    lambda row: row[text_col] if not row['unpacked_highlights'] else row['unpacked_highlights'],\n",
+    "    axis=1\n",
+    ")\n",
+    "\n",
+    "# Define a function to highlight parts of the text\n",
+    "def highlight_text(text, highlights):\n",
+    "    # Ensure highlights is a list of strings\n",
+    "    if isinstance(highlights, str):\n",
+    "        highlights = [highlights]\n",
+    "    # Wrap each highlight in <mark> tags\n",
+    "    for highlight in highlights:\n",
+    "        # Replace highlight text with a highlighted version\n",
+    "        text = text.replace(highlight, f'<mark>{highlight}</mark>')\n",
+    "    return text\n",
+    "\n",
+    "# Define a function to display full details of a row\n",
+    "def show_details(row):\n",
+    "    details = widgets.Output()\n",
+    "    with details:\n",
+    "        # Use inline styles to ensure no scrolling\n",
+    "        content = f\"\"\"\n",
+    "        <div style=\"border: 1px solid black; padding: 10px; max-width: 100%; box-sizing: border-box;\">\n",
+    "            <h3>Details for {row['barcode']}, {row['page']}</h3>\n",
+    "            <p><b>Retrieved text chunk: </b><i>{row[\"unpacked_highlights\"]}</i></p>\n",
+    "            <p>{highlight_text(row[text_col], row[\"unpacked_highlights\"])}</p>\n",
+    "            <img src=\"{row[\"iiif_link\"]}\" alt=\"IIIF Image Preview\" style=\"max-width: 100%; height: auto;\">\n",
+    "            <a href=\"https://digital.onb.ac.at/OnbViewer/viewer.faces?doc=ABO_%2B{row[\"barcode\"]}\" target=\"_blank\">Open ÖNB Viewer</a>\n",
+    "        </div>\n",
+    "        \"\"\"\n",
+    "        display(HTML(content))\n",
+    "    return details\n",
+    "\n",
+    "# Define a function to create a row with more information and a clickable button\n",
+    "def create_row(row, idx):\n",
+    "    button = widgets.Button(description=\"Inspect\", layout=widgets.Layout(width=\"auto\"))\n",
+    "\n",
+    "    # When the button is clicked, show more details\n",
+    "    def on_button_clicked(b):\n",
+    "        details_box.clear_output()\n",
+    "        with details_box:\n",
+    "            display(show_details(row))\n",
+    "    \n",
+    "    button.on_click(on_button_clicked)\n",
+    "    \n",
+    "    # Highlight the text in the 'unpacked_highlights' column\n",
+    "    unpacked = highlight_text(row['unpacked_highlights'], row['unpacked_highlights'])\n",
+    "    \n",
+    "    # Create a display row with button, highlights, and iiif links\n",
+    "    row_display = widgets.HBox([\n",
+    "        widgets.HTML(value=f\"<b>{idx + 1}</b>\"),            # Display the index + 1\n",
+    "        widgets.HTML(\n",
+    "            value=f\"<i>{row['document']}: </i>{unpacked}\",\n",
+    "            layout=widgets.Layout(width='500px')\n",
+    "        ),                          # Highlights as text\n",
+    "        widgets.HTML(\n",
+    "            value=f\"Score: {row['_score']:.3f}\",\n",
+    "            layout=widgets.Layout(width='100px')\n",
+    "        ),  # Score as text\n",
+    "        widgets.HTML(\n",
+    "            value=f\"Prev. ranks: {row['rank']}\",\n",
+    "            layout=widgets.Layout(width='100px')\n",
+    "        ),  # Ranks as text\n",
+    "        widgets.HTML(\n",
+    "            value=f\"Corpus: {row['corpus']}\",\n",
+    "            layout=widgets.Layout(width='100px')\n",
+    "        ),  # Corpus as text\n",
+    "        button                                                 # Button to view more details\n",
+    "    ])\n",
+    "    \n",
+    "    # Wrap the row display in a Box with a border\n",
+    "    boxed_row_display = widgets.Box(\n",
+    "        [row_display],\n",
+    "        layout=widgets.Layout(\n",
+    "            border='1px solid black',  # Set the border style\n",
+    "            padding='10px',            # Add padding inside the box\n",
+    "            margin='5px 0px',          # Add margin above and below the box\n",
+    "            width='100%'               # Ensure the box spans the full width\n",
+    "        )\n",
+    "    )\n",
+    "    \n",
+    "    return boxed_row_display\n",
+    "\n",
+    "# Create an interactive list of display rows (one for each DataFrame row)\n",
+    "rows = [create_row(row, idx) for idx, row in final_scores_df.head(R).iterrows()]\n",
+    "\n",
+    "# Display area for detailed view\n",
+    "details_box = widgets.Output()\n",
+    "details_box.layout = widgets.Layout(\n",
+    "    border='1px solid black',\n",
+    "    padding='10px',\n",
+    "    width='100%',  # Ensure full width\n",
+    "    height='auto',  # Adjust height automatically\n",
+    ")\n",
+    "\n",
+    "# Display the rows and details box\n",
+    "rows_box = widgets.VBox(rows)\n",
+    "display(rows_box, details_box)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88e0078b",
+   "metadata": {},
+   "source": [
+    "# Similarity Metrics with Sentence Transformer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 84,
+   "id": "c783b576",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sentence_transformers import SentenceTransformer, util"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 85,
+   "id": "b5db773f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\vignolim\\Anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load a pre-trained Sentence Transformer model\n",
+    "model = SentenceTransformer('paraphrase-MiniLM-L6-v2')\n",
+    "\n",
+    "# List of sentences\n",
+    "list_of_sentences = final_scores_df['unpacked_highlights'].tolist()\n",
+    "\n",
+    "# Step 1: Encode the sentences and the query into embeddings\n",
+    "sentence_embeddings = model.encode(list_of_sentences, convert_to_tensor=True)\n",
+    "query_embedding = model.encode(query, convert_to_tensor=True)\n",
+    "\n",
+    "# Step 2: Compute the cosine similarities between the query and the list of sentences\n",
+    "cosine_scores = util.pytorch_cos_sim(query_embedding, sentence_embeddings)\n",
+    "\n",
+    "# Step 3: Create a pandas DataFrame with sentences and their corresponding cosine similarity scores\n",
+    "df_cosine_sim = pd.DataFrame({\n",
+    "    'Sentence': list_of_sentences,\n",
+    "    'Cosine Similarity': cosine_scores.squeeze().tolist()\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 86,
+   "id": "33ef4dfa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Sentence</th>\n",
+       "      <th>Cosine Similarity</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>S. sono. Gmelin hat diesen Vogel in der 13.</td>\n",
+       "      <td>0.587100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Ich todtete eine gelbe Bachstelze ***) und ein...</td>\n",
+       "      <td>0.293896</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Auf diesem Damme stehen von Zeit zu Zeit klein...</td>\n",
+       "      <td>0.347629</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Indessen schienen mir diese Vögel sich mehr de...</td>\n",
+       "      <td>0.471382</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Die Wanderung dieser Vögel nach Ägypten hat je...</td>\n",
+       "      <td>0.476594</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>Ich traf auch eine Wachtel an, die ich sogleic...</td>\n",
+       "      <td>0.200288</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>Diese Vögel sind sehr fett, ihr Fleisch ist za...</td>\n",
+       "      <td>0.407871</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>Reichhaltig war er. Kein Vogel ist in Ägypten ...</td>\n",
+       "      <td>0.515706</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>Linn. Den ersten Morgens bei einem sehr schöne...</td>\n",
+       "      <td>0.310032</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>Bearbeitung der Gegenden um den Canal Salza. F...</td>\n",
+       "      <td>0.271253</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>Diese Vögel hielten sich nicht in der Nähe von</td>\n",
+       "      <td>0.515144</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>nennt. Diese letztgenannten Vögel sprangen paa...</td>\n",
+       "      <td>0.336441</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>Binther- Nilaat- Schildfröste- te- Vogel- Raub...</td>\n",
+       "      <td>0.163589</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>Durch diese habe ich auch den Unterschied des ...</td>\n",
+       "      <td>0.268166</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>Sie sagten uns, diese Vögel kommen oft nach Ab...</td>\n",
+       "      <td>0.320704</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>Es hielt daher sehr schwer, sich einen von die...</td>\n",
+       "      <td>0.352532</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>Senesbaum Vogel Beschreibung einer Art von Fal...</td>\n",
+       "      <td>0.317160</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>Diese Vögel gehören nicht zu einerlei Art. Die...</td>\n",
+       "      <td>0.433839</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>Ich erkannte Beccafien, Feldlerchen und Sperli...</td>\n",
+       "      <td>0.303594</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>Kapitel. Natron -- Bleichen der Leinwand und d...</td>\n",
+       "      <td>0.265617</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Sentence  Cosine Similarity\n",
+       "0         S. sono. Gmelin hat diesen Vogel in der 13.           0.587100\n",
+       "1   Ich todtete eine gelbe Bachstelze ***) und ein...           0.293896\n",
+       "2   Auf diesem Damme stehen von Zeit zu Zeit klein...           0.347629\n",
+       "3   Indessen schienen mir diese Vögel sich mehr de...           0.471382\n",
+       "4   Die Wanderung dieser Vögel nach Ägypten hat je...           0.476594\n",
+       "5   Ich traf auch eine Wachtel an, die ich sogleic...           0.200288\n",
+       "6   Diese Vögel sind sehr fett, ihr Fleisch ist za...           0.407871\n",
+       "7   Reichhaltig war er. Kein Vogel ist in Ägypten ...           0.515706\n",
+       "8   Linn. Den ersten Morgens bei einem sehr schöne...           0.310032\n",
+       "9   Bearbeitung der Gegenden um den Canal Salza. F...           0.271253\n",
+       "10     Diese Vögel hielten sich nicht in der Nähe von           0.515144\n",
+       "11  nennt. Diese letztgenannten Vögel sprangen paa...           0.336441\n",
+       "12  Binther- Nilaat- Schildfröste- te- Vogel- Raub...           0.163589\n",
+       "13  Durch diese habe ich auch den Unterschied des ...           0.268166\n",
+       "14  Sie sagten uns, diese Vögel kommen oft nach Ab...           0.320704\n",
+       "15  Es hielt daher sehr schwer, sich einen von die...           0.352532\n",
+       "16  Senesbaum Vogel Beschreibung einer Art von Fal...           0.317160\n",
+       "17  Diese Vögel gehören nicht zu einerlei Art. Die...           0.433839\n",
+       "18  Ich erkannte Beccafien, Feldlerchen und Sperli...           0.303594\n",
+       "19  Kapitel. Natron -- Bleichen der Leinwand und d...           0.265617"
+      ]
+     },
+     "execution_count": 86,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_cosine_sim.head(R)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 87,
+   "id": "c26885b7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add SentenceTransformer Cosine Similarity score to output dataframe\n",
+    "final_scores_df['ST_cosine_similarity'] = df_cosine_sim['Cosine Similarity']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 88,
+   "id": "17d5abeb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add direct link to ÖNB Viewer to output dataframe\n",
+    "base_url = \"https://digital.onb.ac.at/OnbViewer/viewer.faces?doc=ABO_%2B\"\n",
+    "\n",
+    "# Create the 'onb_viewer_link' column by appending each 'barcode' to the base URL\n",
+    "final_scores_df['onb_viewer_link'] = base_url + final_scores_df['barcode'].astype(str)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 89,
+   "id": "b6a2d7e0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>document</th>\n",
+       "      <th>rrf_score</th>\n",
+       "      <th>barcode</th>\n",
+       "      <th>page</th>\n",
+       "      <th>iiif_link</th>\n",
+       "      <th>text_orig</th>\n",
+       "      <th>text_clean</th>\n",
+       "      <th>text_prep</th>\n",
+       "      <th>_id</th>\n",
+       "      <th>_highlights</th>\n",
+       "      <th>_score</th>\n",
+       "      <th>rank</th>\n",
+       "      <th>corpus</th>\n",
+       "      <th>rerank</th>\n",
+       "      <th>unpacked_highlights</th>\n",
+       "      <th>ST_cosine_similarity</th>\n",
+       "      <th>onb_viewer_link</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Z166069305_252</td>\n",
+       "      <td>0.032258</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>252</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>224  7  Dieſer Falke war ein Weibchen. Der Ein...</td>\n",
+       "      <td>Dieser Falke war ein Weibchen. Der Eingeweide ...</td>\n",
+       "      <td>Dieser Falke war ein Weibchen. Der Eingeweidek...</td>\n",
+       "      <td>8a5893c6-fde6-42f3-94e3-16db95629fe5</td>\n",
+       "      <td>[{'text_prep': 'S. sono. Gmelin hat diesen Vog...</td>\n",
+       "      <td>0.856740</td>\n",
+       "      <td>2</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>1</td>\n",
+       "      <td>S. sono. Gmelin hat diesen Vogel in der 13.</td>\n",
+       "      <td>0.587100</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Z166069305_43</td>\n",
+       "      <td>0.031025</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>43</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>16  5  1  entgegen, indem er mit großer Heftig...</td>\n",
+       "      <td>entgegen, indem er mit groser Heftigkeit nach ...</td>\n",
+       "      <td>entgegen, indem er mit großer Heftigkeit nach ...</td>\n",
+       "      <td>4205ec0a-9cf3-448d-afba-3023e4c92052</td>\n",
+       "      <td>[{'text_prep': 'Ich todtete eine gelbe Bachste...</td>\n",
+       "      <td>0.852868</td>\n",
+       "      <td>3/6</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Ich todtete eine gelbe Bachstelze ***) und ein...</td>\n",
+       "      <td>0.293896</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Z166069305_265</td>\n",
+       "      <td>0.029911</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>265</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>237  Seeufer, an welchem wir bis zum See Maadi...</td>\n",
+       "      <td>Seeufer, an welchem wir bis zum See Maadie hin...</td>\n",
+       "      <td>Seeufer, an welchem wir bis zum See Maadie hin...</td>\n",
+       "      <td>5fc895c1-b587-4442-a34c-dd446c3054f2</td>\n",
+       "      <td>[{'text_prep': 'Auf diesem Damme stehen von Ze...</td>\n",
+       "      <td>0.848943</td>\n",
+       "      <td>4/10</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>3</td>\n",
+       "      <td>Auf diesem Damme stehen von Zeit zu Zeit klein...</td>\n",
+       "      <td>0.347629</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Z166069305_344</td>\n",
+       "      <td>0.029412</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>344</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>316  niederſchoß, wünſchte ich mir Glük, dieſe...</td>\n",
+       "      <td>niederschos, wunschte ich mir Gluk, diesen uns...</td>\n",
+       "      <td>Niederschoss, wünschte ich mir Glück, diesen u...</td>\n",
+       "      <td>a7debf83-ac70-445a-afee-5f35515b3e1d</td>\n",
+       "      <td>[{'text_prep': 'Indessen schienen mir diese Vö...</td>\n",
+       "      <td>0.843653</td>\n",
+       "      <td>8</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>4</td>\n",
+       "      <td>Indessen schienen mir diese Vögel sich mehr de...</td>\n",
+       "      <td>0.471382</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Z166069305_243</td>\n",
+       "      <td>0.027778</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>243</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>215  don -einſam liegenden Seen beleben, bleib...</td>\n",
+       "      <td>don -einsam liegenden Seen beleben, bleiben di...</td>\n",
+       "      <td>don-einsam liegenden Seen beleben, bleiben die...</td>\n",
+       "      <td>58487609-e48c-47a1-8a0a-06569fef7f53</td>\n",
+       "      <td>[{'text_prep': 'Die Wanderung dieser Vögel nac...</td>\n",
+       "      <td>0.835293</td>\n",
+       "      <td>24/3</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>5</td>\n",
+       "      <td>Die Wanderung dieser Vögel nach Ägypten hat je...</td>\n",
+       "      <td>0.476594</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>413</th>\n",
+       "      <td>Z166069305_70</td>\n",
+       "      <td>0.002119</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>70</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>42  ohne Thellung und ohne Abgaben die Früchte...</td>\n",
+       "      <td>ohne Thellung und ohne Abgaben die Fruchte ihr...</td>\n",
+       "      <td>ohne Thellung und ohne Abgaben die Früchte ihr...</td>\n",
+       "      <td>06af6d12-1ddf-492d-8848-2537dc8b4a9c</td>\n",
+       "      <td>[{'text_prep': 'Vorzüglich zog Frankreich groß...</td>\n",
+       "      <td>0.794844</td>\n",
+       "      <td>412</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>414</td>\n",
+       "      <td>Vorzüglich zog Frankreich große Vorteile darau...</td>\n",
+       "      <td>0.237914</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>414</th>\n",
+       "      <td>Z166069305_299</td>\n",
+       "      <td>0.002114</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>299</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>271  i  Auf meiner ganzen Reife habe ich und m...</td>\n",
+       "      <td>Auf meiner ganzen Reife habe ich und meine Rei...</td>\n",
+       "      <td>Auf meiner ganzen Reise habe ich und meine Gef...</td>\n",
+       "      <td>01d37f8c-4a15-42a2-9ef5-a73fc7739766</td>\n",
+       "      <td>[{'text_prep': 'Der Fluss sieht rotlich und en...</td>\n",
+       "      <td>0.794709</td>\n",
+       "      <td>413</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>415</td>\n",
+       "      <td>Der Fluss sieht rotlich und endlich grünlich a...</td>\n",
+       "      <td>0.106832</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>415</th>\n",
+       "      <td>Z166069305_410</td>\n",
+       "      <td>0.002110</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>410</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>382  ilmy  1  \"  mann, der feſt von dieſem Ged...</td>\n",
+       "      <td>ilmy mann, der fest von diesem Gedanken uberze...</td>\n",
+       "      <td>Ilmymann, der fest von diesem Gedanken überzeu...</td>\n",
+       "      <td>dde560fc-9473-4faa-9e3c-1280ba131213</td>\n",
+       "      <td>[{'text_prep': 'Sie hielten mich in ihrer Einb...</td>\n",
+       "      <td>0.791636</td>\n",
+       "      <td>414</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>416</td>\n",
+       "      <td>Sie hielten mich in ihrer Einbildung für einen...</td>\n",
+       "      <td>0.140597</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>416</th>\n",
+       "      <td>Z166069305_75</td>\n",
+       "      <td>0.002105</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>75</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>47  3  dle ich hier gemacht habe, werden iðre ...</td>\n",
+       "      <td>dle ich hier gemacht habe, werden ire Stelle i...</td>\n",
+       "      <td>Die ich hier gemacht habe, werden ihre Stelle ...</td>\n",
+       "      <td>32dd47f1-c7f4-4d4a-bb50-f428f79fce3b</td>\n",
+       "      <td>[{'text_prep': 'Die ich hier gemacht habe, wer...</td>\n",
+       "      <td>0.791505</td>\n",
+       "      <td>415</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>417</td>\n",
+       "      <td>Die ich hier gemacht habe, werden ihre Stelle ...</td>\n",
+       "      <td>0.136045</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>417</th>\n",
+       "      <td>Z166069305_6</td>\n",
+       "      <td>0.002101</td>\n",
+       "      <td>Z166069305</td>\n",
+       "      <td>6</td>\n",
+       "      <td>https://iiif.onb.ac.at/images/ABO/Z166069305/0...</td>\n",
+       "      <td>2125 murid  gobiothers  conale  ܪܝ  ، فرد. برد...</td>\n",
+       "      <td>2125 murid gobiothers conale KOENISE KAISERLIC...</td>\n",
+       "      <td>2255 Murdoch  Gouverneurs Comte KOENIGSE KAISE...</td>\n",
+       "      <td>21bb54fb-3e4e-456c-9605-f89970db24b7</td>\n",
+       "      <td>[{'text_prep': '2255 Murdoch  Gouverneurs Comt...</td>\n",
+       "      <td>0.791373</td>\n",
+       "      <td>416</td>\n",
+       "      <td>D19</td>\n",
+       "      <td>418</td>\n",
+       "      <td>2255 Murdoch  Gouverneurs Comte KOENIGSE KAISE...</td>\n",
+       "      <td>0.109442</td>\n",
+       "      <td>https://digital.onb.ac.at/OnbViewer/viewer.fac...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>418 rows × 17 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           document  rrf_score     barcode page  \\\n",
+       "0    Z166069305_252   0.032258  Z166069305  252   \n",
+       "1     Z166069305_43   0.031025  Z166069305   43   \n",
+       "2    Z166069305_265   0.029911  Z166069305  265   \n",
+       "3    Z166069305_344   0.029412  Z166069305  344   \n",
+       "4    Z166069305_243   0.027778  Z166069305  243   \n",
+       "..              ...        ...         ...  ...   \n",
+       "413   Z166069305_70   0.002119  Z166069305   70   \n",
+       "414  Z166069305_299   0.002114  Z166069305  299   \n",
+       "415  Z166069305_410   0.002110  Z166069305  410   \n",
+       "416   Z166069305_75   0.002105  Z166069305   75   \n",
+       "417    Z166069305_6   0.002101  Z166069305    6   \n",
+       "\n",
+       "                                             iiif_link  \\\n",
+       "0    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "1    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "2    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "3    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "4    https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "..                                                 ...   \n",
+       "413  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "414  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "415  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "416  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "417  https://iiif.onb.ac.at/images/ABO/Z166069305/0...   \n",
+       "\n",
+       "                                             text_orig  \\\n",
+       "0    224  7  Dieſer Falke war ein Weibchen. Der Ein...   \n",
+       "1    16  5  1  entgegen, indem er mit großer Heftig...   \n",
+       "2    237  Seeufer, an welchem wir bis zum See Maadi...   \n",
+       "3    316  niederſchoß, wünſchte ich mir Glük, dieſe...   \n",
+       "4    215  don -einſam liegenden Seen beleben, bleib...   \n",
+       "..                                                 ...   \n",
+       "413  42  ohne Thellung und ohne Abgaben die Früchte...   \n",
+       "414  271  i  Auf meiner ganzen Reife habe ich und m...   \n",
+       "415  382  ilmy  1  \"  mann, der feſt von dieſem Ged...   \n",
+       "416  47  3  dle ich hier gemacht habe, werden iðre ...   \n",
+       "417  2125 murid  gobiothers  conale  ܪܝ  ، فرد. برد...   \n",
+       "\n",
+       "                                            text_clean  \\\n",
+       "0    Dieser Falke war ein Weibchen. Der Eingeweide ...   \n",
+       "1    entgegen, indem er mit groser Heftigkeit nach ...   \n",
+       "2    Seeufer, an welchem wir bis zum See Maadie hin...   \n",
+       "3    niederschos, wunschte ich mir Gluk, diesen uns...   \n",
+       "4    don -einsam liegenden Seen beleben, bleiben di...   \n",
+       "..                                                 ...   \n",
+       "413  ohne Thellung und ohne Abgaben die Fruchte ihr...   \n",
+       "414  Auf meiner ganzen Reife habe ich und meine Rei...   \n",
+       "415  ilmy mann, der fest von diesem Gedanken uberze...   \n",
+       "416  dle ich hier gemacht habe, werden ire Stelle i...   \n",
+       "417  2125 murid gobiothers conale KOENISE KAISERLIC...   \n",
+       "\n",
+       "                                             text_prep  \\\n",
+       "0    Dieser Falke war ein Weibchen. Der Eingeweidek...   \n",
+       "1    entgegen, indem er mit großer Heftigkeit nach ...   \n",
+       "2    Seeufer, an welchem wir bis zum See Maadie hin...   \n",
+       "3    Niederschoss, wünschte ich mir Glück, diesen u...   \n",
+       "4    don-einsam liegenden Seen beleben, bleiben die...   \n",
+       "..                                                 ...   \n",
+       "413  ohne Thellung und ohne Abgaben die Früchte ihr...   \n",
+       "414  Auf meiner ganzen Reise habe ich und meine Gef...   \n",
+       "415  Ilmymann, der fest von diesem Gedanken überzeu...   \n",
+       "416  Die ich hier gemacht habe, werden ihre Stelle ...   \n",
+       "417  2255 Murdoch  Gouverneurs Comte KOENIGSE KAISE...   \n",
+       "\n",
+       "                                      _id  \\\n",
+       "0    8a5893c6-fde6-42f3-94e3-16db95629fe5   \n",
+       "1    4205ec0a-9cf3-448d-afba-3023e4c92052   \n",
+       "2    5fc895c1-b587-4442-a34c-dd446c3054f2   \n",
+       "3    a7debf83-ac70-445a-afee-5f35515b3e1d   \n",
+       "4    58487609-e48c-47a1-8a0a-06569fef7f53   \n",
+       "..                                    ...   \n",
+       "413  06af6d12-1ddf-492d-8848-2537dc8b4a9c   \n",
+       "414  01d37f8c-4a15-42a2-9ef5-a73fc7739766   \n",
+       "415  dde560fc-9473-4faa-9e3c-1280ba131213   \n",
+       "416  32dd47f1-c7f4-4d4a-bb50-f428f79fce3b   \n",
+       "417  21bb54fb-3e4e-456c-9605-f89970db24b7   \n",
+       "\n",
+       "                                           _highlights    _score  rank corpus  \\\n",
+       "0    [{'text_prep': 'S. sono. Gmelin hat diesen Vog...  0.856740     2    D19   \n",
+       "1    [{'text_prep': 'Ich todtete eine gelbe Bachste...  0.852868   3/6    D19   \n",
+       "2    [{'text_prep': 'Auf diesem Damme stehen von Ze...  0.848943  4/10    D19   \n",
+       "3    [{'text_prep': 'Indessen schienen mir diese Vö...  0.843653     8    D19   \n",
+       "4    [{'text_prep': 'Die Wanderung dieser Vögel nac...  0.835293  24/3    D19   \n",
+       "..                                                 ...       ...   ...    ...   \n",
+       "413  [{'text_prep': 'Vorzüglich zog Frankreich groß...  0.794844   412    D19   \n",
+       "414  [{'text_prep': 'Der Fluss sieht rotlich und en...  0.794709   413    D19   \n",
+       "415  [{'text_prep': 'Sie hielten mich in ihrer Einb...  0.791636   414    D19   \n",
+       "416  [{'text_prep': 'Die ich hier gemacht habe, wer...  0.791505   415    D19   \n",
+       "417  [{'text_prep': '2255 Murdoch  Gouverneurs Comt...  0.791373   416    D19   \n",
+       "\n",
+       "     rerank                                unpacked_highlights  \\\n",
+       "0         1        S. sono. Gmelin hat diesen Vogel in der 13.   \n",
+       "1         2  Ich todtete eine gelbe Bachstelze ***) und ein...   \n",
+       "2         3  Auf diesem Damme stehen von Zeit zu Zeit klein...   \n",
+       "3         4  Indessen schienen mir diese Vögel sich mehr de...   \n",
+       "4         5  Die Wanderung dieser Vögel nach Ägypten hat je...   \n",
+       "..      ...                                                ...   \n",
+       "413     414  Vorzüglich zog Frankreich große Vorteile darau...   \n",
+       "414     415  Der Fluss sieht rotlich und endlich grünlich a...   \n",
+       "415     416  Sie hielten mich in ihrer Einbildung für einen...   \n",
+       "416     417  Die ich hier gemacht habe, werden ihre Stelle ...   \n",
+       "417     418  2255 Murdoch  Gouverneurs Comte KOENIGSE KAISE...   \n",
+       "\n",
+       "     ST_cosine_similarity                                    onb_viewer_link  \n",
+       "0                0.587100  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "1                0.293896  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "2                0.347629  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "3                0.471382  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "4                0.476594  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "..                    ...                                                ...  \n",
+       "413              0.237914  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "414              0.106832  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "415              0.140597  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "416              0.136045  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "417              0.109442  https://digital.onb.ac.at/OnbViewer/viewer.fac...  \n",
+       "\n",
+       "[418 rows x 17 columns]"
+      ]
+     },
+     "execution_count": 89,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_scores_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "id": "931d1a7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save DataFrame to CSV\n",
+    "final_scores_df.to_csv('data/sonnini_llm_corrected/i_onit-sonnini-DHd2025-prep-q_Pferd, Pferde.csv', index=False)\n",
+    "\n",
+    "# Save DataFrame to Excel\n",
+    "final_scores_df.to_excel('data/sonnini_llm_corrected/i_onit-sonnini-DHd2025-prep-q_Pferd, Pferde.xlsx', index=False, engine='openpyxl')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4f4a0175",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/indexing/index_data.py

@@ -0,0 +1,97 @@
+"""
+This script creates a Marqo index of preprocessed and original OCR texts. Each page is indexed as a document that is split into 2 sentences long vectors.
+The model used for sentence embedding is https://huggingface.co/flax-sentence-embeddings/all_datasets_v4_mpnet-base.
+
+Code by Michela Vignoli. Parts of this code were developed with assistance from Simon König.
+"""
+
+from pprint import pprint
+import csv
+import marqo as mq
+
+##
+## Connect to Marqo
+##
+
+MARQO_URL = "http://10.103.251.104:8882"
+marqoClient = mq.Client(url=MARQO_URL)
+#pprint(marqoClient.get_indexes())
+
+##
+## Index settings
+##
+
+settings = {
+    "textPreprocessing": {
+        "splitLength": 2,
+        "splitOverlap": 0,
+        "splitMethod": "sentence",
+    },
+}
+
+##
+## Ask if index exists, if not create it
+##
+
+indexName = "onit-sonnini-DHd2025-clean"
+print("Indexname: ", indexName)
+current_indexes = [d["indexName"] for d in marqoClient.get_indexes()["results"]]
+if indexName in current_indexes:
+    print(f"Index already exists: {indexName} ")
+    # Set indexName as the current index
+    print(f"Defaulting to index connection. Index connected: {indexName} ")
+else:  # Create a new index
+    print(f"Index does not exist: {indexName} ")
+    print(f"Creating index: {indexName} ")
+    marqoClient.create_index(
+        indexName,
+        model="flax-sentence-embeddings/all_datasets_v4_mpnet-base",
+        settings_dict=settings
+    )
+
+## List of models integrated in Marqo: https://docs.marqo.ai/latest/models/marqo/list-of-models/
+
+pprint(marqoClient.get_indexes())
+
+##
+## Load dict of data
+##
+
+
+# Load list of dictionaries with each dictionary containing keys: text, barcode, page
+# CSV path
+csv_file = 'data/DHd_index-cleaned.csv'
+
+# Read data from CSV file into a list of dictionaries
+with open(csv_file, mode='r', encoding='utf-8') as file:
+    reader = csv.DictReader(file)
+    animal_descriptions = [row for row in reader]
+
+# Function to clean text by replacing \n with spaces
+def clean_text(text):
+    return text.replace('\n', ' ').strip()
+
+# Clean the 'text' field in each dictionary
+for entry in animal_descriptions:
+    entry['text_orig'] = clean_text(entry['text_orig'])
+    entry['text_clean'] = clean_text(entry['text_clean'])
+    entry['text_prep'] = clean_text(entry['text_prep'])
+
+pprint(animal_descriptions[:3])
+
+##
+## Add documents to the index
+##
+
+print(f"Indexing data...")
+# Define client_batch_size
+client_batch_size = 128
+
+# Indexing
+marqoClient.index(indexName).add_documents(
+    animal_descriptions,
+    client_batch_size=client_batch_size,
+    tensor_fields=["text_clean"],
+)
+
+print(f"Data has been indexed in {indexName}")

src/preprocessing/clean_books.py

@@ -0,0 +1,80 @@
+"""
+This script cleans the OCR files, so that we have uniform documents with the same pre-processing applied to each of
+them. For every book, a new document is created so that the original file is always available for cross-checking etc.
+
+Code adapted from Travelogues project, by Jan Rörden. Source: https://github.com/travelogues/scripts/blob/master/groundtruth/
+
+"""
+
+import os
+import re
+import string
+import unicodedata
+from tqdm import tqdm
+
+
+# directories
+books_original_dir = 'source/path/'
+output_dir = 'output/path/'
+
+# Ensure the cleaned directory exists
+os.makedirs(output_dir, exist_ok=True)
+
+# Function to remove accents and umlauts
+def remove_accents(input_str):
+    # Normalize to decompose accents
+    nfkd_form = unicodedata.normalize('NFKD', input_str)
+    # Filter out diacritical marks
+    return ''.join([c for c in nfkd_form if not unicodedata.combining(c)])
+
+for fname in tqdm(sorted(os.listdir(books_original_dir))):
+    # Save the current id for file naming later
+    current_book_id = fname[:-4]
+
+    # Process only .txt files
+    if fname.endswith('.txt'):
+        with open(os.path.join(books_original_dir, fname), 'r', encoding='utf-8') as f:
+            cleaned_lines = []
+            page_lines = []
+
+            for line in f:
+                # Replace long s and ß with normal s
+                clean_line = re.sub(r'[ſß]', 's', line)
+
+                # Remove accents and umlauts
+                clean_line = remove_accents(clean_line)
+
+                # Remove all non-word characters except whitespace and punctuation
+                clean_line = re.sub(r'[^a-zA-Z0-9\s' + re.escape(string.punctuation) + ']', '', clean_line)
+
+                # Convert to lowercase
+                #clean_line = clean_line.lower()
+
+                # Strip trailing spaces but keep line breaks
+                clean_line = clean_line.rstrip()
+
+                # Exclude lines based on criteria
+                if len(clean_line) < 3 or clean_line.isdigit() or not re.search(r'[a-zA-Z]', clean_line):
+                    continue  # Skip the line
+
+                # Check for a new page indicated by a blank line
+                if clean_line == "":
+                    # Handle empty pages
+                    if not page_lines or page_lines[0].startswith('statuscode') or page_lines[0].startswith('<html>'):
+                        cleaned_lines.append("<empty page>")
+                    else:
+                        cleaned_lines.extend(page_lines)
+                    page_lines = []
+                else:
+                    page_lines.append(clean_line)
+
+            # Handle the last page if the file ends without a blank line
+            if not page_lines or page_lines[0].startswith('statuscode') or page_lines[0].startswith('<html>'):
+                cleaned_lines.append("<empty page>")
+            else:
+                cleaned_lines.extend(page_lines)
+
+        # Save the cleaned text to a new file, retaining line breaks
+        cleaned_file_path = os.path.join(output_dir, f"{current_book_id}_cleaned.txt")
+        with open(cleaned_file_path, 'w', encoding='utf-8') as cleaned_file:
+            cleaned_file.write('\n'.join(cleaned_lines))  # Write lines with original line breaks

src/preprocessing/llm_keywords.py

@@ -0,0 +1,120 @@
+"""
+This script parses through the cleaned texts and prompts Ollama to create lists of keywords of the contents from each page while focusing on particular topics of interest.
+It was tested with German texts from the 17th, 18th, and 19th century.
+
+Code by Michela Vignoli. Parts of this code were developed with assistance from GPT-4 and GPT-3 (free version).
+"""
+
+import json
+import subprocess
+import os
+import time
+
+def get_data(root_folder, extension='.txt'):
+    data = []
+    # Walk through all folders and files in the root directory
+    for folder, _, files in os.walk(root_folder):
+        for file in files:
+            if file.endswith(extension):
+                file_path = os.path.join(folder, file)
+                filename = os.path.splitext(os.path.basename(file_path))[0]
+                folder_path = os.path.join(f'preprocessed/{folder}_keywords')
+                #print(f"Found file: {file_path}")
+                
+                # Read the file content
+                with open(file_path, 'r', encoding="utf-8") as f:
+                    text = f.read()
+                    data.append({"path": folder_path, "text": text, "filename": filename})
+    return data
+
+def extract_corrected_text(raw_response):
+    # Split the response by newline to handle multiple JSON objects
+    lines = raw_response.strip().split('\n')
+    response_segments = []
+    
+    # Process each line as a JSON object
+    for line in lines:
+        try:
+            json_obj = json.loads(line)
+            response_segment = json_obj.get('response', '')
+            response_segments.append(response_segment)
+        except json.JSONDecodeError:
+            print(f"Skipping invalid JSON line: {line}")
+    
+    # Join all response segments into a single text
+    full_text = ''.join(response_segments)
+    return full_text
+
+def correct_text_with_llm(text, retries=3):
+    print('-------------------------')
+    print('Processing LLM request...')
+    prompt = "You are a historian expert in historical German texts. From the following faulty OCR texts generated from historical traveolgues printed from the 17-19th century, create a list of keywords summarizing the animals, plants, landscapes, and maps mentioned in the text. If animals, plants, landscapes, or maps are mentioned in the text, include the aforementioned general keywords and more specific ones afterwards. If none of these is mentioned in the text, add up to three keywords describing the content of the page.\n\n" + text
+
+    # Prepare the cURL command
+    curl_command = [
+        'curl',
+        'http://your.ip:port/api/generate',
+        '-d', json.dumps({
+            "model": "llama3.1:70b",
+            "prompt": prompt
+        }),
+        '-H', 'Content-Type: application/json'
+    ]
+
+    for attempt in range(retries):
+        print(f"Attempt {attempt + 1} of {retries}: Processing LLM request...")
+        result = subprocess.run(curl_command, capture_output=True, text=True, encoding='utf-8')
+        
+        if result.returncode != 0:
+            print(f"Error: cURL command failed with error: {result.stderr}")
+            time.sleep(2)  # Wait before retrying
+            continue
+        
+        corrected_text = extract_corrected_text(result.stdout)
+        print("Corrected text:", corrected_text)
+        
+        if corrected_text:
+            return corrected_text
+        
+        print("Invalid response, retrying...")
+        time.sleep(2)  # Wait before retrying
+
+    print(f"Failed to get a valid response from the LLM API.")
+    return None
+
+
+def process_txt(root_folder):
+    # Read text files from the root folder
+    data = get_data(root_folder)
+    
+    # Process each file in the data list
+    for item in data:
+        text = item["text"]
+        folder_path = item["path"]
+        filename = item["filename"]
+        
+        # Define the output path for the corrected text file
+        output_path = os.path.join(folder_path, f"{filename}_keywords.txt")
+        
+        # Create the necessary directories if they don't exist
+        os.makedirs(folder_path, exist_ok=True)
+        
+        # Apply the LLM correction function
+        corrected_text = correct_text_with_llm(text)
+        
+        # Determine the appropriate output path based on whether correction succeeded
+        if corrected_text:
+            # Save the corrected text into a .txt file
+            with open(output_path, "w", encoding="utf-8") as file:
+                file.write(corrected_text)
+            print(f"Processed and corrected text saved to {output_path}")
+        else:
+            # Save the original text into a .txt file with a different name if correction failed
+            output_path = os.path.join(folder_path, f"{filename}_FAILED.txt")
+            with open(output_path, "w", encoding="utf-8") as file:
+                file.write(text)
+            print(f"Failed to correct text. Original text saved to {output_path}")
+
+# Example usage
+root_folder = 'source/path/'
+process_txt(root_folder)

src/preprocessing/llm_preprocessing.py

@@ -0,0 +1,120 @@
+"""
+This script parses through the cleaned texts and prompts Ollama to create transcriptions of the faulty OCR texts staying as close as possible to the original wording.
+It was tested with German texts from the 17th, 18th, and 19th century.
+
+Code by Michela Vignoli. Parts of this code were developed with assistance from GPT-4 and GPT-3 (free version).
+"""
+
+import json
+import subprocess
+import os
+import time
+
+def get_data(root_folder, extension='.txt'):
+    data = []
+    # Walk through all folders and files in the root directory
+    for folder, _, files in os.walk(root_folder):
+        for file in files:
+            if file.endswith(extension):
+                file_path = os.path.join(folder, file)
+                filename = os.path.splitext(os.path.basename(file_path))[0]
+                folder_path = os.path.join(f'preprocessed/{folder}_preprocessed')
+                #print(f"Found file: {file_path}")
+                
+                # Read the file content with detected encoding
+                with open(file_path, 'r', encoding="utf-8") as f:
+                    text = f.read()
+                    data.append({"path": folder_path, "text": text, "filename": filename})
+    return data
+
+def extract_corrected_text(raw_response):
+    # Split the response by newline to handle multiple JSON objects
+    lines = raw_response.strip().split('\n')
+    response_segments = []
+    
+    # Process each line as a JSON object
+    for line in lines:
+        try:
+            json_obj = json.loads(line)
+            response_segment = json_obj.get('response', '')
+            response_segments.append(response_segment)
+        except json.JSONDecodeError:
+            print(f"Skipping invalid JSON line: {line}")
+    
+    # Join all response segments into a single text
+    full_text = ''.join(response_segments)
+    return full_text
+
+def correct_text_with_llm(text, retries=3):
+    print('-------------------------')
+    print('Processing LLM request...')
+    prompt = "You are a historian expert in historical German texts. Correct the following faulty OCR texts generated from historical traveolgues printed from the 17-19th century. Remain as closely to the original, historical wording as possible while correcting all the errors from the faulty OCR. Output the corrected text by removing the unnecessary line breaks in the pages, where full sentences occur. Leave the line breaks in other pages. Only output the corrected text without further comments, explanations, or information. Omit Corrected text: before the actual text.\n\n" + text
+
+    # Prepare the cURL command
+    curl_command = [
+        'curl',
+        'http://your.ip:port/api/generate',
+        '-d', json.dumps({
+            "model": "llama3.1:70b",
+            "prompt": prompt
+        }),
+        '-H', 'Content-Type: application/json'
+    ]
+
+    for attempt in range(retries):
+        print(f"Attempt {attempt + 1} of {retries}: Processing LLM request...")
+        result = subprocess.run(curl_command, capture_output=True, text=True, encoding='utf-8')
+        
+        if result.returncode != 0:
+            print(f"Error: cURL command failed with error: {result.stderr}")
+            time.sleep(2)  # Wait before retrying
+            continue
+        
+        corrected_text = extract_corrected_text(result.stdout)
+        print("Corrected text:", corrected_text)
+        
+        if corrected_text:
+            return corrected_text
+        
+        print("Invalid response, retrying...")
+        time.sleep(2)  # Wait before retrying
+
+    print(f"Failed to get a valid response from the LLM API.")
+    return None
+
+
+def process_txt(root_folder):
+    # Read text files from the root folder
+    data = get_data(root_folder)
+    
+    # Process each file in the data list
+    for item in data:
+        text = item["text"]
+        folder_path = item["path"]
+        filename = item["filename"]
+        
+        # Define the output path for the corrected text file
+        output_path = os.path.join(folder_path, f"{filename}_corrected.txt")
+        
+        # Create the necessary directories if they don't exist
+        os.makedirs(folder_path, exist_ok=True)
+        
+        # Apply the LLM correction function
+        corrected_text = correct_text_with_llm(text)
+        
+        # Determine the appropriate output path based on whether correction succeeded
+        if corrected_text:
+            # Save the corrected text into a .txt file
+            with open(output_path, "w", encoding="utf-8") as file:
+                file.write(corrected_text)
+            print(f"Processed and corrected text saved to {output_path}")
+        else:
+            # Save the original text into a .txt file with a different name if correction failed
+            output_path = os.path.join(folder_path, f"{filename}_FAILED.txt")
+            with open(output_path, "w", encoding="utf-8") as file:
+                file.write(text)
+            print(f"Failed to correct text. Original text saved to {output_path}")
+
+# Example usage
+root_folder = 'source/folder/'
+process_txt(root_folder)

src/utils/annotations_preprocessing.py

@@ -0,0 +1,92 @@
+"""
+This script matches the text annotations created on the original OCR files to the cleaned version of the text.
+The annotations were created with Recogito https://recogito.pelagios.org/.
+
+Code by Michela Vignoli. Parts of this code were developed with assistance from GPT-4 and GPT-3 (free version).
+"""
+
+## Import packages ##
+
+import pandas as pd
+import os
+import re
+from typing import Union
+
+
+## Import annotations from Recogito ##
+
+path_1 = "source/path/"
+filename_1 = 'jiggvn0g5pgx34.csv'
+
+# Function to reformat the labels
+def reformat_labels(label_str):
+    labels = str(label_str).split('|')  # Split the string by '|'
+    reformatted = ', '.join([f"'{label}'" for label in labels])  # Enclose each label in ''
+    return reformatted
+
+df1 = pd.read_csv(os.path.join(path_1, filename_1))[["UUID", "FILE", "QUOTE_TRANSCRIPTION", "ANCHOR", "COMMENTS", "TAGS"]]
+
+# Apply the function to the 'labels' column
+df1['TAGS'] = df1['TAGS'].apply(reformat_labels)
+
+
+## Extract page numbers from merged OCR text file ##
+
+# Read the entire text file into a single string
+with open('source/path/Z255430508_clean_merged.txt', 'r', encoding='utf-8') as file1:
+    text_content1 = file1.read()
+
+
+# Function to find a number in the preceding character sequence
+def find_number_before_position(text: str, position: int, search_length: int = 10000) -> Union[str, str]:
+    """
+    Finds the last number following 'page' in the text preceding or succeeding the given position.
+
+    Parameters:
+    - text (str): The full text to search within.
+    - position (int): The position in the text to search around.
+    - search_length (int): The length of text to search before or after the position.
+
+    Returns:
+    - Union[str, str]: The last number found after 'page' in the preceding or succeeding text,
+                        or a warning message "Warning: No matches found. Check!" if no match is found.
+    """
+    if not isinstance(text, str):
+        raise ValueError("text must be a string")
+    if not isinstance(position, int) or position < 0:
+        raise ValueError("position must be a non-negative integer")
+    if not isinstance(search_length, int) or search_length < 0:
+        raise ValueError("search_length must be a non-negative integer")
+    
+    if position == 0:
+        # Search after the position
+        following_text = text[position:position + search_length]
+        matches = re.findall(r'page(\d+)', following_text)
+        if matches:
+            return matches[0]  # Return the first match found
+        else:
+            return "Check!"
+    else:
+        # Search before the position
+        start_position = max(0, position - search_length)
+        preceding_text = text[start_position:position]
+        matches = re.findall(r'page(\d+)', preceding_text)
+        if matches:
+            return matches[-1]  # Return the last match found
+        else:
+            return "Check!"
+
+# Apply the function to each row in the DataFrames
+df1['PAGE'] = pd.to_numeric(df1['ANCHOR'].str.extract(r'(\d+)')[0], errors='coerce').apply(lambda x: find_number_before_position(text_content1, x))
+
+
+## Annotation analysis ##
+
+# Split the labels by '|' and flatten the list of lists into a single list
+all_labels = df1['TAGS'].str.split(', ').sum()
+
+# Count the occurrences of each label
+label_counts = pd.Series(all_labels).value_counts()
+
+print(label_counts)
+

src/utils/extract_data.py

@@ -0,0 +1,128 @@
+"""
+This script creates a CSV with all data to be indexed on the Marqo server.
+
+Code by Michela Vignoli. Parts of this code were developed with assistance from GPT-4 and GPT-3 (free version).
+"""
+
+import os
+import csv
+import chardet
+from tqdm import tqdm
+
+# Helper function to get all file paths with a specific extension in a folder
+def collect_files(folder, extension=".txt"):
+    file_paths = []
+    for root, _, files in os.walk(folder):
+        for file in files:
+            if file.endswith(extension):
+                file_paths.append(os.path.join(root, file))
+    return file_paths
+
+# Function to process files and extract their text
+def process_file(file_path):
+    try:
+        # Detect encoding
+        with open(file_path, 'rb') as f:
+            result = chardet.detect(f.read())
+            encoding = result['encoding']
+
+        # Read the file
+        with open(file_path, 'r', encoding=encoding) as f:
+            return f.read()
+    except Exception as e:
+        print(f"Error processing {file_path}: {e}")
+        return None
+
+# Combine data from clean, orig, and prep folders
+def combine_data(clean_files, orig_files, prep_files):
+    combined_data = []
+
+    # Index files by (barcode, page) for matching
+    def index_files(files):
+        indexed = {}
+        for file in files:
+            barcode = os.path.basename(os.path.dirname(file))[:10]
+            page = os.path.basename(file)[:5]
+            indexed[(barcode, page)] = file
+        return indexed
+
+    clean_index = index_files(clean_files)
+    orig_index = index_files(orig_files)
+    prep_index = index_files(prep_files)
+
+    # Process files and combine data
+    for key in tqdm(clean_index.keys(), desc="Combining data", unit="file"):
+        clean_file = clean_index.get(key)
+        orig_file = orig_index.get(key)
+        prep_file = prep_index.get(key)
+
+        # Extract text
+        text_clean = process_file(clean_file) if clean_file else None
+        text_orig = process_file(orig_file) if orig_file else None
+        text_prep = process_file(prep_file) if prep_file else None
+
+        # Add combined data row
+        barcode, page = key
+        page_url = page[:5].zfill(8)
+        iiif_link = f"https://iiif.onb.ac.at/images/ABO/{barcode}/{page_url}/full/full/0/native.jpg"
+
+        combined_data.append({
+            "barcode": barcode,
+            "page": page,
+            "iiif_link": iiif_link,
+            "text_clean": text_clean,
+            "text_orig": text_orig,
+            "text_prep": text_prep,
+        })
+
+    return combined_data
+
+# Lists of folders to process
+clean_folders = [
+    'source/path/DHd 2025 dataset/Sonnini Z166069305/Z166069305_clean/',
+]
+orig_folders = [
+    "source/path/02-texts/D19/Z166069305",
+]
+prep_folders = [
+    'source/path/DHd 2025 dataset/Sonnini Z166069305/Z166069305_clean_preprocessed/',
+]
+
+# Collect file paths
+clean_files = [file for folder in clean_folders for file in collect_files(folder)]
+orig_files = [file for folder in orig_folders for file in collect_files(folder)]
+prep_files = [file for folder in prep_folders for file in collect_files(folder)]
+
+# Combine data from all folders
+all_data = combine_data(clean_files, orig_files, prep_files)
+
+# Specify the file path and create the directory if it does not exist
+csv_file = 'output/path/DHd_index.csv'
+os.makedirs(os.path.dirname(csv_file), exist_ok=True)
+
+# Write data to CSV file
+with open(csv_file, mode='w', newline='', encoding='utf-8') as file:
+    writer = csv.DictWriter(file, fieldnames=["barcode", "page", "iiif_link", "text_clean", "text_orig", "text_prep"])
+    writer.writeheader()
+    writer.writerows(all_data)
+
+#### IMPORTANT ####
+#### Data Cleaning Needed after storing the file ####
+
+"""
+# Clean data
+# Specify columns to check and update
+columns_to_check = ["text_clean", "text_prep"]
+
+# Check for rows where any of the columns contain "status code" or "empty page"
+rows_to_update = index_DHd[columns_to_check].applymap(lambda x: any(keyword in str(x) for keyword in ["status code", "empty page"])).any(axis=1)
+
+# Replace content in the specified columns for the identified rows
+index_DHd.loc[rows_to_update, columns_to_check] = "<empty page>
+
+# Remove artifacts from the LLM generation process
+index_DHd['text_prep'] = index_DHd['text_prep'].str.strip("Here is the corrected text:")
+
+""""
+
+print(f"Data from all folders has been written to {csv_file}")