Added a multi word search tool on the redaction review tool page

app.py

@@ -12,7 +12,7 @@ from tools.data_anonymise import anonymise_data_files
 from tools.auth import authenticate_user
 from tools.load_spacy_model_custom_recognisers import custom_entities
 from tools.custom_csvlogger import CSVLogger_custom
-from tools.find_duplicate_pages import run_duplicate_analysis, exclude_match, handle_selection_and_preview, apply_whole_page_redactions_from_list, create_annotation_objects_from_duplicates
+from tools.find_duplicate_pages import run_duplicate_analysis, exclude_match, handle_selection_and_preview, apply_whole_page_redactions_from_list, create_annotation_objects_from_duplicates, run_full_search_and_analysis
 from tools.textract_batch_call import analyse_document_with_textract_api, poll_whole_document_textract_analysis_progress_and_download, load_in_textract_job_details, check_for_provided_job_id, check_textract_outputs_exist, replace_existing_pdf_input_for_whole_document_outputs
 
 # Suppress downcasting warnings
@@ -409,8 +409,15 @@ with app:
                 with gr.Tab("Search and redact"):
                     with gr.Accordion("Search text", open=True):
                         with gr.Row(equal_height=True):
-                            page_entity_dropdown_redaction = gr.Dropdown(label="Page", value="1", allow_custom_value=True)                    
-                            reset_dropdowns_btn_new = gr.Button(value="Reset page filter")
+                            page_entity_dropdown_redaction = gr.Dropdown(label="Page", value="1", allow_custom_value=True, scale=4)                    
+                            reset_dropdowns_btn_new = gr.Button(value="Reset page filter", scale=1)
+
+                        with gr.Row():
+                            multi_word_search_text = gr.Textbox(label="Multi-word text search", value="", scale=4)
+                            multi_word_search_text_btn = gr.Button(value="Search", scale=1)
+
+                        with gr.Accordion("Search options", open=False):
+                            similarity_search_score_minimum = gr.Number(value=1.0, minimum=0.4, maximum=1.0, label="Minimum similarity score for match (max=1)")
 
                         all_page_line_level_ocr_results_with_words_df = gr.Dataframe(pd.DataFrame(data={"page":[], "line":[], "word_text":[], "word_x0":[],	"word_y0":[],"word_x1":[],"word_y1":[]}), type="pandas", label="Click table row to select and go to page", headers=["page", "line", "word_text", "word_x0","word_y0","word_x1","word_y1"], show_fullscreen_button=True, wrap=False, max_height=400, show_search="filter")
                                                                     
@@ -472,6 +479,7 @@ with app:
             with gr.Row():
                 results_df_preview = gr.Dataframe(
                     label="Similarity Results",
+                    headers=["Page1_File",	"Page1_Start_Page",	"Page1_End_Page",	"Page2_File",	"Page2_Start_Page",	"Page2_End_Page",	"Match_Length",	"Avg_Similarity",	"Page1_Text",	"Page2_Text"],
                     wrap=True,
                     show_fullscreen_button=True,
                     show_search=True,
@@ -636,7 +644,7 @@ with app:
     success(fn=check_for_relevant_ocr_output_with_words, inputs=[doc_file_name_no_extension_textbox, text_extract_method_radio, output_folder_textbox], outputs=[relevant_ocr_output_with_words_found_checkbox])
 
     # Run redaction function
-    document_redact_btn.click(fn = reset_state_vars, outputs=[all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, textract_metadata_textbox, annotator, output_file_list_state, log_files_output_list_state, recogniser_entity_dataframe, recogniser_entity_dataframe_base, pdf_doc_state, duplication_file_path_outputs_list_state, redaction_output_summary_textbox, is_a_textract_api_call, textract_query_number]).\
+    document_redact_btn.click(fn = reset_state_vars, outputs=[all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, textract_metadata_textbox, annotator, output_file_list_state, log_files_output_list_state, recogniser_entity_dataframe, recogniser_entity_dataframe_base, pdf_doc_state, duplication_file_path_outputs_list_state, redaction_output_summary_textbox, is_a_textract_api_call, textract_query_number, all_page_line_level_ocr_results_with_words]).\
         success(fn= enforce_cost_codes, inputs=[enforce_cost_code_textbox, cost_code_choice_drop, cost_code_dataframe_base]).\
         success(fn= choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, text_extract_method_radio, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_num, redaction_output_summary_textbox, output_file_list_state, log_files_output_list_state, first_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_df, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base],
                     outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state], api_name="redact_doc")
@@ -690,7 +698,7 @@ with app:
         success(fn = prepare_image_or_pdf, inputs=[output_review_files, text_extract_method_radio, all_page_line_level_ocr_results_df_base, all_page_line_level_ocr_results_with_words_df_base, latest_file_completed_num, redaction_output_summary_textbox, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox, input_folder_textbox, prepare_images_bool_false], outputs=[redaction_output_summary_textbox, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_df, document_cropboxes, page_sizes, textract_output_found_checkbox, all_img_details_state, all_page_line_level_ocr_results_df_base, relevant_ocr_output_with_words_found_checkbox,  all_page_line_level_ocr_results_with_words_df_base], api_name="prepare_doc").\
         success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_df, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, page_sizes, all_image_annotations_state])
     
-    # Manual updates to review di
+    # Manual updates to review df
     review_file_df.input(update_annotator_page_from_review_df, inputs=[review_file_df, images_pdf_state, page_sizes, all_image_annotations_state, annotator, selected_entity_dataframe_row, input_folder_textbox, doc_full_file_name_textbox], outputs=[annotator, all_image_annotations_state, annotate_current_page, page_sizes, review_file_df, annotate_previous_page]).\
         success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_df, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, page_sizes, all_image_annotations_state])
 
@@ -782,6 +790,32 @@ with app:
     ###
     page_entity_dropdown_redaction.select(update_redact_choice_df_from_page_dropdown, inputs=[page_entity_dropdown_redaction, all_page_line_level_ocr_results_with_words_df_base], outputs=[all_page_line_level_ocr_results_with_words_df])
 
+    multi_word_search_text.submit(
+    fn=run_full_search_and_analysis,
+    inputs=[
+        multi_word_search_text,
+        all_page_line_level_ocr_results_with_words_df_base,
+        similarity_search_score_minimum
+    ],
+    outputs=[
+        all_page_line_level_ocr_results_with_words_df,
+        duplicate_files_out,
+        full_duplicate_data_by_file
+    ])
+
+    multi_word_search_text_btn.click(
+    fn=run_full_search_and_analysis,
+    inputs=[
+        multi_word_search_text,
+        all_page_line_level_ocr_results_with_words_df_base,
+        similarity_search_score_minimum
+    ],
+    outputs=[
+        all_page_line_level_ocr_results_with_words_df,
+        duplicate_files_out,
+        full_duplicate_data_by_file
+    ])
+
     # Clicking on a cell in the redact items table will take you to that page
     all_page_line_level_ocr_results_with_words_df.select(df_select_callback_dataframe_row_ocr_with_words, inputs=[all_page_line_level_ocr_results_with_words_df], outputs=[selected_entity_dataframe_row_redact, selected_entity_dataframe_row_text_redact]).\
         success(update_annotator_page_from_review_df, inputs=[review_file_df, images_pdf_state, page_sizes, all_image_annotations_state, annotator, selected_entity_dataframe_row_redact, input_folder_textbox, doc_full_file_name_textbox], outputs=[annotator, all_image_annotations_state, annotate_current_page, page_sizes, review_file_df, annotate_previous_page]).\
@@ -871,7 +905,7 @@ with app:
         ],
         outputs=[
             results_df_preview,
-            duplicate_files_out,            
+            duplicate_files_out, 
             full_duplicate_data_by_file
         ]
     )
@@ -896,6 +930,8 @@ with app:
         outputs=[review_file_df, all_image_annotations_state]).\
         success(update_annotator_page_from_review_df, inputs=[review_file_df, images_pdf_state, page_sizes, all_image_annotations_state, annotator, selected_entity_dataframe_row, input_folder_textbox, doc_full_file_name_textbox], outputs=[annotator, all_image_annotations_state, annotate_current_page, page_sizes, review_file_df, annotate_previous_page]).\
         success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_df, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, page_sizes, all_image_annotations_state])
+    
+    
 
     ###
     # SETTINGS PAGE INPUT / OUTPUT
@@ -910,6 +946,7 @@ with app:
     in_deny_list_state.input(update_dataframe, inputs=[in_deny_list_state], outputs=[in_deny_list_state])
     in_fully_redacted_list_state.input(update_dataframe, inputs=[in_fully_redacted_list_state], outputs=[in_fully_redacted_list_state])
 
+    # Apply whole page redactions from the provided whole page redaction csv file upload/list of specific page numbers given by user
     apply_fully_redacted_list_btn.click(
         fn=apply_whole_page_redactions_from_list,
         inputs=[in_fully_redacted_list_state, doc_file_name_with_extension_textbox, review_file_df, duplicate_files_out, pdf_doc_state, page_sizes, all_image_annotations_state],

tools/file_redaction.py

@@ -104,7 +104,7 @@ def choose_and_run_redactor(file_paths:List[str],
  page_min:int=0,
  page_max:int=999,
  estimated_time_taken_state:float=0.0,
- handwrite_signature_checkbox:List[str]=["Extract handwriting", "Extract signatures"],
+ handwrite_signature_checkbox:List[str]=list(["Extract handwriting", "Extract signatures"]),
  all_request_metadata_str:str = "",
  annotations_all_pages:List[dict]=list(),
  all_page_line_level_ocr_results_df:pd.DataFrame=None,
@@ -132,7 +132,7 @@ def choose_and_run_redactor(file_paths:List[str],
  ocr_file_path:str="",
  all_page_line_level_ocr_results:list[dict] = list(),
  all_page_line_level_ocr_results_with_words:list[dict] = list(),
- all_page_line_level_ocr_results_with_words_df:pd.DataFrame=list(),
+ all_page_line_level_ocr_results_with_words_df:pd.DataFrame=None,
  prepare_images:bool=True,
  RETURN_PDF_END_OF_REDACTION:bool=RETURN_PDF_END_OF_REDACTION,
  progress=gr.Progress(track_tqdm=True)):
@@ -202,6 +202,11 @@ def choose_and_run_redactor(file_paths:List[str],
     all_textract_request_metadata = all_request_metadata_str.split('\n') if all_request_metadata_str else []
     review_out_file_paths = [prepared_pdf_file_paths[0]]
 
+    print("all_page_line_level_ocr_results_with_words at start of choose and run...:", all_page_line_level_ocr_results_with_words)
+
+    if all_page_line_level_ocr_results_with_words_df is None:
+         all_page_line_level_ocr_results_with_words_df = pd.DataFrame()
+
     # Create copies of out_file_path objects to avoid overwriting each other on append actions
     out_file_paths = out_file_paths.copy()
     log_files_output_paths = log_files_output_paths.copy()
@@ -663,6 +668,9 @@ def choose_and_run_redactor(file_paths:List[str],
                 if all_page_line_level_ocr_results_with_words_df_file_path not in log_files_output_paths:
                     log_files_output_paths.append(all_page_line_level_ocr_results_with_words_df_file_path)
 
+                if all_page_line_level_ocr_results_with_words_df_file_path not in out_file_paths:
+                    out_file_paths.append(all_page_line_level_ocr_results_with_words_df_file_path)
+
             # Convert the gradio annotation boxes to relative coordinates
             progress(0.93, "Creating review file output")
             page_sizes = page_sizes_df.to_dict(orient="records")
@@ -1203,7 +1211,7 @@ def redact_page_with_pymupdf(page:Page, page_annotations:dict, image:Image=None,
 # IMAGE-BASED OCR PDF TEXT DETECTION/REDACTION WITH TESSERACT OR AWS TEXTRACT
 ###
 
-def merge_img_bboxes(bboxes, combined_results: Dict, page_signature_recogniser_results=[], page_handwriting_recogniser_results=[], handwrite_signature_checkbox: List[str]=["Extract handwriting", "Extract signatures"], horizontal_threshold:int=50, vertical_threshold:int=12):
+def merge_img_bboxes(bboxes:list, combined_results: Dict, page_signature_recogniser_results=[], page_handwriting_recogniser_results=[], handwrite_signature_checkbox: List[str]=["Extract handwriting", "Extract signatures"], horizontal_threshold:int=50, vertical_threshold:int=12):
 
     all_bboxes = []
     merged_bboxes = []
@@ -1385,6 +1393,8 @@ def redact_image_pdf(file_path:str,
 
     tic = time.perf_counter()
 
+    print("all_page_line_level_ocr_results_with_words in redact_image_pdf:", all_page_line_level_ocr_results_with_words)
+
     file_name = get_file_name_without_type(file_path)    
     comprehend_query_number_new = 0
 

tools/find_duplicate_pages.py

@@ -1,10 +1,12 @@
 import pandas as pd
 import os
 import re
+import itertools # For getting file pairs
+import numpy as np # For efficient index finding
 from tools.helper_functions import OUTPUT_FOLDER
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
-from typing import List, Tuple, Optional, Dict
+from typing import List, Tuple, Optional, Dict, Union
 from collections import defaultdict
 import gradio as gr
 from gradio import Progress
@@ -16,76 +18,400 @@ import en_core_web_lg
 nlp = en_core_web_lg.load()
 
 similarity_threshold = 0.95
+number_of_zeros_to_add_to_index = 7 # Number of zeroes to add between page number and line numbers to get a unique page/line index value
+ID_MULTIPLIER = 100000
+
+def extract_indices_from_page_ranges(
+    results_df: pd.DataFrame,
+    start_col: str = 'Page2_Start_Page',
+    end_col: str = 'Page2_End_Page',
+    modulo_divisor_number_of_zeros: int = number_of_zeros_to_add_to_index, # Search for number of added
+    converted_index: bool = False # Has the index been converted to the page_no + 0000 + line number format that needs the modulo divisor to convert back?
+) -> List[int]:
+    all_indices = set()
+    modulo_divisor = int("1" + modulo_divisor_number_of_zeros*"0")
+
+    for _, row in results_df.iterrows():
+        start_page = row[start_col]
+        end_page = row[end_col]
+        for encoded_page_id in range(start_page, end_page + 1):
+            if converted_index == True:
+                original_page, original_index = _parse_page_line_id(encoded_page_id)#(encoded_page_id % modulo_divisor) - 1
+            else:
+                original_index = encoded_page_id
+
+            all_indices.add(original_index)
+    return sorted(list(all_indices))
 
-def combine_ocr_output_text(input_files:List[str], combine_pages:bool=True, output_folder:str=OUTPUT_FOLDER):
+def run_full_search_and_analysis(
+    search_query_text: str,
+    word_level_df_orig: pd.DataFrame,
+    similarity_threshold: float = 1,
+    combine_pages: bool = False,    
+    min_word_count: int = 1,
+    min_consecutive_pages: int = 1,
+    greedy_match: bool = True,
+    remake_index: bool = False,
+    progress=gr.Progress(track_tqdm=True)
+):
     """
-    Combines text from multiple CSV files containing page and text columns.
-    Groups text by file and page number, concatenating text within these groups.
+    This function orchestrates the entire pipeline for finding duplicate pages based on a user's search query. It takes in the search query text, the original word-level OCR data, and various parameters to control the analysis. The function then:
+
+    1. Converts the user's search query into a DataFrame format suitable for analysis.
+    2. Prepares the main word-level OCR data for processing by converting it into the required format.
+    3. Combines the search query DataFrame with the prepared OCR data DataFrame.
+    4. Executes the similarity analysis on the combined data using the specified parameters such as similarity threshold, minimum word count, minimum consecutive pages, and greedy match strategy.
+
+    Parameters:
+    - search_query_text (str): The text entered by the user to search for in the OCR data.
+    - word_level_df_orig (pd.DataFrame): The original DataFrame containing word-level OCR data.
+    - similarity_threshold (float, optional): The minimum similarity score required for two pages to be considered duplicates. Defaults to 1.
+    - combine_pages (bool, optional): A flag indicating whether to combine text from the same page number within a file. Defaults to False.    
+    - min_word_count (int, optional): The minimum number of words required for a page to be considered in the analysis. Defaults to 1.
+    - min_consecutive_pages (int, optional): The minimum number of consecutive pages required to be considered a match. Defaults to 1.
+    - greedy_match (bool, optional): A flag indicating whether to use a greedy strategy for matching consecutive pages. Defaults to True.
+    - remake_index (bool, optional): A flag indicating whether to remake the index of the DataFrame during processing. Defaults to False.
+    - progress (gr.Progress, optional): A Progress object to track the progress of the operation. Defaults to a Progress object with track_tqdm set to True.
+    """
+
+    if len(search_query_text) < 3:
+        raise Warning("Please use a search query with at least three letters.")
+    if len(search_query_text) > 100:
+        raise Warning("Please use a search query with at less than 100 characters.")
+
+    # Step 1: Process the user's search query string
+    search_query_data, query_word_length = create_dataframe_from_string(search_query_text, split_words=True)
+    if not search_query_data:
+        # Handle case where user submits an empty search string
+        raise Warning("Could not convert search string to required format") 
+
+    if query_word_length > 10:
+        # Handle case where user submits an empty search string
+        raise Warning("Please use a query with less than 10 words") 
+
+    # Overwrite min_consecutive_pages with the search string length
+    min_consecutive_pages = query_word_length  
     
-    Args:
-        input_files (list): List of paths to CSV files
+    # Create word index from reference table
+    word_level_df_orig["index"] = word_level_df_orig.index
+    word_level_df = word_level_df_orig.copy() 
+
+    # Step 2: Process the main word-level OCR DataFrame
+    word_level_data = convert_word_level_df(word_level_df, file_name="source_document")
+
+    # Step 3: Combine both data sources into one list
+    all_data_to_process = search_query_data + word_level_data
+    if not all_data_to_process:
+        raise gr.Error("No data to process. Please check your inputs.")
     
+    # Step 4: Run the combination logic
+    combined_df, _, full_out_ocr_df = combine_ocr_dataframes(
+        input_data=all_data_to_process,
+        combine_pages=combine_pages,
+        output_folder=None, # No need to save this intermediate file
+        remake_index=remake_index
+    )
+
+    # Step 5: Run the final similarity analysis on the combined data
+    results_df, duplicate_files, full_data = identify_similar_text_sequences(
+        df_combined=combined_df,
+        similarity_threshold=similarity_threshold,
+        min_word_count=min_word_count,
+        min_consecutive_pages=min_consecutive_pages,
+        greedy_match=greedy_match,
+        combine_pages=combine_pages,
+        inter_file_only=True,
+        do_text_clean=False,
+        progress=progress
+    )
+
+    print("Finished text search")
+
+    # Map the results back to the reference data file
+    if remake_index == True:
+        results_df_index_list = extract_indices_from_page_ranges(results_df, converted_index=True)
+    else:
+        results_df_index_list = extract_indices_from_page_ranges(results_df, converted_index=False)
+
+    word_level_df_out = word_level_df_orig.loc[word_level_df_orig["index"].isin(results_df_index_list)]
+
+    return word_level_df_out, duplicate_files, full_data
+
+def create_all_data_to_process(converted_data:pd.DataFrame, other_data_list:List[Tuple]):
+    all_data_to_process = converted_data + other_data_list
+    return all_data_to_process
+
+def convert_word_level_df(
+    word_level_df: pd.DataFrame,
+    file_name: str = "converted_dataframe"
+) -> List[Tuple[str, pd.DataFrame]]:
+    """
+    Converts a word-level OCR DataFrame to the format for
+    combine_ocr_dataframes.
+
+    A simple renaming and selection of relevant columns
+
+    Args:
+        word_level_df (pd.DataFrame):
+            A DataFrame containing detailed OCR output. Must include at least
+            the columns: 'page', 'line', and 'word_text'.
+        file_name (str, optional):
+            A unique identifier or "dummy" filename to assign to the resulting
+            data. Defaults to "converted_dataframe".
+
     Returns:
-        pd.DataFrame: Combined dataframe with columns [file, page, text]
+        List[Tuple[str, pd.DataFrame]]:
+            A list containing a single tuple of (file_name, DataFrame), ready
+            to be used as input for the combine_ocr_dataframes function. The
+            DataFrame will have 'page' and 'text' columns.
     """
-    all_data = []
-    output_files = []
+    # --- 1. Validate Input ---
+    required_columns = ['page', 'line', 'word_text']
+    if not all(col in word_level_df.columns for col in required_columns):
+        raise ValueError(f"Input DataFrame must contain all of the following columns: {required_columns}")
 
-    if isinstance(input_files, str):
-        file_paths_list = [input_files]
+    df = word_level_df.copy()
+
+    # --- 2. Process the DataFrame ---
+    # Ensure word_text is a string to allow for joining
+    df['word_text'] = df['word_text'].astype(str)
+
+    # Group by page and line number, then join the words with a space (not needed for word level search)
+    # The result is a Series with a MultiIndex (page, line)
+    #line_text_series = df.groupby(['page', 'line'])['word_text'].apply(' '.join)
+
+    # Convert the Series back to a DataFrame and reset the index
+    #line_level_df = line_text_series.reset_index()
+
+    # Rename the aggregated column from 'word_text' to the required 'text'
+    df = df.rename(columns={'word_text': 'text'})
+
+    # --- 3. Finalise the structure ---
+    # We now have a DataFrame with columns [page, line, text].
+    final_df = df[['page', 'text']]
+
+    # --- 4. Package for output ---
+    # Return in the required List[Tuple[str, DataFrame]] format
+    return [(file_name, final_df)]
+
+def create_dataframe_from_string(
+    text_string: str,
+    file_name: str = "user_search_query",
+    page_number: int = 1,
+    split_words: bool = False
+) -> Tuple[List[Tuple[str, pd.DataFrame]], int]:
+    """
+    Converts a string into a DataFrame compatible with combine_ocr_dataframes.
+
+    Can operate in two modes:
+    1. As a single-line document (default).
+    2. As a multi-line document where each word from the string is a separate line.
+
+    Args:
+        text_string (str): The input text to be placed in the DataFrame.
+        file_name (str, optional): A dummy filename to assign to this text.
+                                   Defaults to "user_search_query".
+        page_number (int, optional): A dummy page number to assign. Defaults to 1.
+        split_words (bool, optional): If True, splits the input string by
+                                      whitespace and creates a row for each word.
+                                      If False (default), the entire string is
+                                      treated as a single text entry.
+
+    Returns:
+        Tuple[List[Tuple[str, pd.DataFrame]], int]:
+            A list containing a single tuple: (file_name, DataFrame).
+            The DataFrame has 'page' and 'text' columns. Also, an integer value indicating the number of words in the search string.
+            Returns an empty list if the input string is empty or whitespace.
+    """
+    # Handle empty input gracefully, this works for both modes.
+    if not text_string or not text_string.strip():
+        print("Warning: Input string is empty. Returning an empty list.")
+        return [], 0
+
+    if split_words:
+        # --- MODE 2: Split string into words, one per row ---
+        words = text_string.split()
+        len_words = len(words)
+        data = {
+            # Assign the same page number to every word
+            'page': [page_number] * len(words),
+            # The list of words becomes the text column
+            'text': words
+        }
     else:
-        file_paths_list = input_files
-    
-    for file in file_paths_list:
+        # --- MODE 1: Original behavior, entire string in one row ---
+        len_words = 1
+        data = {
+            'page': [page_number],
+            'text': [text_string]
+        }
 
-        if isinstance(file, str):
-            file_path = file
-        else:
-            file_path = file.name
+    # Create the DataFrame from the prepared data
+    df = pd.DataFrame(data)
 
-        # Read CSV file
-        df = pd.read_csv(file_path)
-        
-        # Ensure required columns exist
+    df["line"] = df.index + 1
+
+    # Return it in the required format: a list containing one (name, df) tuple
+    return [(file_name, df)], len_words
+
+def combine_ocr_dataframes(
+    input_data: List[Tuple[str, pd.DataFrame]],
+    combine_pages: bool = True,
+    output_folder: str = OUTPUT_FOLDER,
+    output_filename: str = "combined_ocr_output.csv",
+    number_of_added_zeros: int = number_of_zeros_to_add_to_index,
+    remake_index:bool = True
+) -> Tuple[pd.DataFrame, List[str]]:
+    """
+    Combines text from multiple pandas DataFrames containing page and text columns.
+
+    This function takes a list of (name, DataFrame) tuples, processes each DataFrame
+    by grouping and concatenating text, and then combines them into a single DataFrame.
+
+    Args:
+        input_data (List[Tuple[str, pd.DataFrame]]):
+            A list of tuples, where each tuple contains a unique identifier (like a filename)
+            and a pandas DataFrame. Each DataFrame must have 'page' and 'text' columns.
+        combine_pages (bool, optional):
+            If True, text from the same page number within a file is joined into a
+            single row. If False, each line of text gets its own row with a unique
+            page identifier. Defaults to True.
+        output_folder (str, optional):
+            The folder where the combined CSV file will be saved. Defaults to OUTPUT_FOLDER.
+        output_filename (str, optional):
+            The name of the output CSV file. Defaults to "combined_ocr_output.csv".
+
+    Returns:
+        Tuple[pd.DataFrame, List[str]]:
+            A tuple containing:
+            - The final combined and processed DataFrame.
+            - A list containing the path to the saved output CSV file.
+    """
+    all_data = []
+
+    for file_identifier, df_initial in input_data:
+        df = df_initial.copy()  # Work on a copy to avoid side effects
+
+        # --- Validation ---
         if 'page' not in df.columns or 'text' not in df.columns:
-            print(f"Warning: Skipping {file_path} - missing required columns 'page' and 'text'")
+            print(f"Warning: Skipping data for '{file_identifier}' - missing required columns 'page' and 'text'.")
             continue
 
+        # --- Processing ---
         df['text'] = df['text'].fillna('').astype(str)
-        
-        # Group by page and concatenate text
-        if combine_pages == True:
-            grouped = df.groupby('page')['text'].apply(' '.join).reset_index()
+
+        if combine_pages:
+            # Group by page and concatenate text into a single string
+            processed_df = df.groupby('page')['text'].apply(' '.join).reset_index()
         else:
-            df['line_number_by_page'] = df.groupby('page').cumcount() + 1
-            df['original_page'] = df['page']
-            df['page'] = df['page'].astype(str).str.zfill(5) + df['line_number_by_page'].astype(str).str.zfill(5)
-            df['page'] = df['page'].astype(int)
+            if remake_index == True:
+                # # Create a unique, sortable page ID for each line without combining
+                # df['line_number_by_page'] = df.groupby('page').cumcount() + 1
+                # df['original_page'] = df['page']
+                # # Create a new page ID that combines page and line number for uniqueness
+                # df['page'] = (
+                #     df['page'].astype(str).str.zfill(number_of_added_zeros) +
+                #     df['line_number_by_page'].astype(str).str.zfill(number_of_added_zeros)
+                # ).astype(int)    
+
+                # Define the multiplier based on the max expected lines per page.
+                # If you expect up to 99,999 lines, use 100,000.
+
+                df['line_number_by_page'] = df.groupby('page').cumcount() + 1
+                df['original_page'] = df['page']
+
+                # Create the new combined ID using arithmetic
+                df['page'] = (df['original_page'] * ID_MULTIPLIER) + df['line_number_by_page'] 
+                            
+            else:
+                if not 'index' in df.columns:
+                    df['index'] = df.index
+                df['page'] = df['index']
+            
+            processed_df = df
+
+        # Add the file identifier column
+        processed_df['file'] = file_identifier
+        all_data.append(processed_df)
 
-            grouped = df #.drop('line_number_by_page', axis=1)
-        
-        # Add filename column
-        grouped['file'] = os.path.basename(file_path)
-        
-        all_data.append(grouped)
-    
     if not all_data:
-        raise ValueError("No valid CSV files were processed")
-    
-    # Combine all dataframes
+        raise ValueError("No valid DataFrames were processed. Ensure input data is not empty and DataFrames have 'page' and 'text' columns.")
+
+    # --- Final Combination ---
     combined_df = pd.concat(all_data, ignore_index=True)
+
+    # Reorder columns to a standard format, dropping intermediate columns
+    final_columns = ['file', 'page', 'text']
+    if 'original_page' in combined_df.columns:
+         final_columns.append('original_page') # Keep for context if created
     
-    # Reorder columns
-    combined_df = combined_df[['file', 'page', 'text']]
+    # Ensure all final columns exist before trying to select them
+    existing_final_columns = [col for col in final_columns if col in combined_df.columns]
 
-    output_combined_file_path = output_folder + "combined_ocr_output_files.csv"
-    combined_df.to_csv(output_combined_file_path, index=None)
+    full_out_ocr_df = combined_df
+    combined_df = combined_df.copy()[existing_final_columns]
 
-    output_files.append(output_combined_file_path)
-    
-    return combined_df, output_files
+    # --- Save Output ---
+    output_files = []
+    if output_folder and output_filename:
+        os.makedirs(output_folder, exist_ok=True)
+        output_path = os.path.join(output_folder, output_filename)
+        combined_df.to_csv(output_path, index=False)
+        output_files.append(output_path)
+        print(f"Successfully combined data and saved to: {output_path}")
+
+    return combined_df, output_files, full_out_ocr_df
+
+def combine_ocr_output_text(
+    input_files: Union[str, List[str]],
+    combine_pages: bool = True,
+    remake_index: bool = True,
+    output_folder: str = OUTPUT_FOLDER
+) -> Tuple[pd.DataFrame, List[str]]:
+    """
+    Reads multiple OCR CSV files, combines them, and saves the result.
+
+    This function serves as a wrapper that reads CSV files from paths and then
+    uses the `combine_ocr_dataframes` function to perform the combination logic.
+
+    Args:
+        input_files (Union[str, List[str]]): A single file path or a list of file paths.
+        combine_pages (bool, optional): See `combine_ocr_dataframes`. Defaults to True.
+        output_folder (str, optional): See `combine_ocr_dataframes`. Defaults to OUTPUT_FOLDER.
+
+    Returns:
+        Tuple[pd.DataFrame, List[str]]: The combined DataFrame and the path to the output file.
+    """
+    if isinstance(input_files, str):
+        file_paths_list = [input_files]
+    else:
+        file_paths_list = input_files
+
+    data_to_process = []
+    for file_path in file_paths_list:
+        try:
+            df = pd.read_csv(file_path)
+            # Use the base filename as the identifier
+            file_identifier = os.path.basename(file_path)
+            data_to_process.append((file_identifier, df))
+        except FileNotFoundError:
+            print(f"Warning: File not found, skipping: {file_path}")
+        except Exception as e:
+            print(f"Warning: Failed to read or process {file_path}. Error: {e}")
 
-def process_data(df:pd.DataFrame, column:str):
+    if not data_to_process:
+        raise ValueError("No valid CSV files could be read or processed.")
+
+    # Call the core function with the loaded data
+    return combine_ocr_dataframes(
+        input_data=data_to_process,
+        combine_pages=combine_pages,
+        output_folder=output_folder,
+        output_filename="combined_ocr_from_files.csv", # Specific name for this path
+        remake_index=remake_index
+    )
+
+def clean_and_stem_text_series(df:pd.DataFrame, column:str):
     '''
     Clean and stem text columns in a data frame
     '''
@@ -213,142 +539,312 @@ def save_results_and_redaction_lists(final_df: pd.DataFrame, output_folder: str,
             
     return output_paths
 
-def identify_similar_pages(
+# def identify_similar_text_sequences(
+#     df_combined: pd.DataFrame,
+#     similarity_threshold: float = 0.9,
+#     min_word_count: int = 10,
+#     min_consecutive_pages: int = 1,
+#     greedy_match: bool = False,
+#     combine_pages:bool=True,
+#     output_folder: str = OUTPUT_FOLDER,
+#     progress=Progress(track_tqdm=True)
+# ) -> Tuple[pd.DataFrame, List[str], pd.DataFrame]:
+#     """
+#     Identifies similar pages with three possible strategies:
+#     1. Single Page: If greedy_match=False and min_consecutive_pages=1.
+#     2. Fixed-Length Subdocument: If greedy_match=False and min_consecutive_pages > 1.
+#     3. Greedy Consecutive Match: If greedy_match=True.
+#     """
+
+#     output_paths = []
+#     progress(0.1, desc="Processing and filtering text")
+#     df = clean_and_stem_text_series(df_combined, 'text')
+#     df['word_count'] = df['text_clean'].str.split().str.len().fillna(0)
+#     original_row_count = len(df)
+#     df_filtered = df[df['word_count'] >= min_word_count].copy()
+#     df_filtered.reset_index(drop=True, inplace=True)
+    
+#     print(f"Filtered out {original_row_count - len(df_filtered)} pages with fewer than {min_word_count} words.")
+
+#     if len(df_filtered) < 2:
+#         return pd.DataFrame(), [], df_combined
+        
+#     vectorizer = TfidfVectorizer()
+#     tfidf_matrix = vectorizer.fit_transform(df_filtered['text_clean'])
+
+#     progress(0.3, desc="Calculating text similarity")
+#     similarity_matrix = cosine_similarity(tfidf_matrix, dense_output=False)
+#     coo_matrix = similarity_matrix.tocoo()
+    
+#     # Create a DataFrame of all individual page pairs above the threshold.
+#     # This is the base for all three matching strategies.
+#     similar_pages = [
+#         (r, c, v) for r, c, v in zip(coo_matrix.row, coo_matrix.col, coo_matrix.data)
+#         if r < c and v >= similarity_threshold
+#     ]
+
+#     if not similar_pages:
+#         return pd.DataFrame(), [], df_combined
+    
+#     base_similarity_df = pd.DataFrame(similar_pages, columns=['Page1_Index', 'Page2_Index', 'Similarity_Score'])
+
+#     progress(0.6, desc="Aggregating results based on matching strategy")
+    
+#     if greedy_match:
+#         print("Finding matches using greedy consecutive strategy.")
+        
+#         # A set of pairs for fast lookups of (page1_idx, page2_idx)
+#         valid_pairs_set = set(zip(base_similarity_df['Page1_Index'], base_similarity_df['Page2_Index']))
+        
+#         # Keep track of indices that have been used in a sequence
+#         consumed_indices_1 = set()
+#         consumed_indices_2 = set()
+        
+#         all_sequences = []
+
+#         # Iterate through all potential starting pairs, sorted for consistent results
+#         sorted_pairs = base_similarity_df.sort_values(['Page1_Index', 'Page2_Index'])
+
+#         for _, row in sorted_pairs.iterrows():
+#             start_idx1, start_idx2 = int(row['Page1_Index']), int(row['Page2_Index'])
+            
+#             # If this pair has already been consumed by a previous sequence, skip it
+#             if start_idx1 in consumed_indices_1 or start_idx2 in consumed_indices_2:
+#                 continue
+
+#             # This is a new sequence, start expanding it
+#             current_sequence = [(start_idx1, start_idx2)]
+#             k = 1
+#             while True:
+#                 next_idx1 = start_idx1 + k
+#                 next_idx2 = start_idx2 + k
+                
+#                 # Check if the next pair in the sequence is a valid match
+#                 if (next_idx1, next_idx2) in valid_pairs_set and \
+#                    next_idx1 not in consumed_indices_1 and \
+#                    next_idx2 not in consumed_indices_2:
+#                     current_sequence.append((next_idx1, next_idx2))
+#                     k += 1
+#                 else:
+#                     # The sequence has ended
+#                     break
+            
+#             # Record the found sequence and mark all its pages as consumed
+#             sequence_indices_1 = [p[0] for p in current_sequence]
+#             sequence_indices_2 = [p[1] for p in current_sequence]
+            
+#             all_sequences.append({
+#                 'Page1_Start_Index': sequence_indices_1[0], 'Page1_End_Index': sequence_indices_1[-1],
+#                 'Page2_Start_Index': sequence_indices_2[0], 'Page2_End_Index': sequence_indices_2[-1],
+#                 'Match_Length': len(current_sequence)
+#             })
+
+#             consumed_indices_1.update(sequence_indices_1)
+#             consumed_indices_2.update(sequence_indices_2)
+
+#         if not all_sequences:
+#             return pd.DataFrame(), [], df_combined
+
+#         subdocument_df = pd.DataFrame(all_sequences)
+
+#     elif min_consecutive_pages > 1:
+#         # --- STRATEGY 2: Fixed-Length Subdocument Matching ---
+#         print(f"Finding consecutive page matches (min_consecutive_pages > 1)")
+#         similarity_df = base_similarity_df.copy()
+#         similarity_df.sort_values(['Page1_Index', 'Page2_Index'], inplace=True)
+#         is_consecutive = (similarity_df['Page1_Index'].diff() == 1) & (similarity_df['Page2_Index'].diff() == 1)
+#         block_id = is_consecutive.eq(False).cumsum()
+#         grouped = similarity_df.groupby(block_id)
+#         agg_results = grouped.agg(
+#             Page1_Start_Index=('Page1_Index', 'first'), Page2_Start_Index=('Page2_Index', 'first'),
+#             Page1_End_Index=('Page1_Index', 'last'), Page2_End_Index=('Page2_Index', 'last'),
+#             Match_Length=('Page1_Index', 'size'), Avg_Similarity=('Similarity_Score', 'mean')
+#         ).reset_index(drop=True)
+#         subdocument_df = agg_results[agg_results['Match_Length'] >= min_consecutive_pages].copy()
+#         if subdocument_df.empty: return pd.DataFrame(), [], df_combined
+
+#     else:
+#         # --- STRATEGY 1: Single Page Matching ---
+#         print(f"Finding single page matches (min_consecutive_pages=1)")
+#         final_df = map_metadata_single_page(base_similarity_df, df_filtered)
+#         # The rest of the logic (saving files) is handled after this if/else block
+#         pass # The final_df is already prepared
+
+#     # --- Map metadata and format output ---
+#     # This block now handles the output for both subdocument strategies (2 and 3)
+#     if greedy_match or min_consecutive_pages > 1:
+#         final_df = map_metadata_subdocument(subdocument_df, df_filtered)
+    
+#     progress(0.8, desc="Saving output files")
+    
+#     output_paths = save_results_and_redaction_lists(final_df, output_folder, combine_pages)
+
+#     return final_df, output_paths, df_combined
+
+def _calculate_inter_file_similarity(df_filtered:pd.DataFrame, vectorizer, similarity_threshold:float, progress:gr.Progress):
+    """
+    Helper function to efficiently calculate similarity ONLY between different files.
+    """
+    print("Calculating inter-file similarity.")
+
+    # Step 1: Fit the vectorizer on the ENTIRE corpus to create a shared vocabulary.
+    # This is crucial for comparing vectors from different files meaningfully.
+    progress(0.2, desc="Building vocabulary...")
+    vectorizer.fit(df_filtered['text_clean'])
+
+    # Step 2: Group the DataFrame by file.
+    file_groups = df_filtered.groupby('file')
+    unique_files = list(file_groups.groups.keys())
+    all_similar_pairs = []
+
+    # Step 3: Iterate through all unique pairs of files.
+    file_combinations = list(itertools.combinations(unique_files, 2))
+    
+    for i, (file1_name, file2_name) in enumerate(progress.tqdm(file_combinations, desc="Comparing file pairs")):
+        group1_df = file_groups.get_group(file1_name)
+        group2_df = file_groups.get_group(file2_name)
+
+        # Step 4: Use the pre-fitted vectorizer to TRANSFORM (not fit_transform) the text of each group.
+        tfidf1 = vectorizer.transform(group1_df['text_clean'])
+        tfidf2 = vectorizer.transform(group2_df['text_clean'])
+
+        # Step 5: Calculate similarity between the two groups.
+        # This is a much smaller matrix (pages_in_file1 x pages_in_file2).
+        similarity_matrix_subset = cosine_similarity(tfidf1, tfidf2)
+
+        # Step 6: Find pairs in this sub-matrix that exceed the threshold.
+        # `np.where` is highly efficient for this.
+        page1_indices, page2_indices = np.where(similarity_matrix_subset >= similarity_threshold)
+
+        # Step 7: Map the local indices back to the original global indices from `df_filtered`.
+        for p1_local_idx, p2_local_idx in zip(page1_indices, page2_indices):
+            global_idx1 = group1_df.index[p1_local_idx]
+            global_idx2 = group2_df.index[p2_local_idx]
+            score = similarity_matrix_subset[p1_local_idx, p2_local_idx]
+            
+            # Ensure r < c convention to match the original method
+            r, c = min(global_idx1, global_idx2), max(global_idx1, global_idx2)
+            all_similar_pairs.append((r, c, score))
+    
+    if not all_similar_pairs:
+        return pd.DataFrame()
+
+    # Create the final DataFrame, which is now pre-filtered.
+    return pd.DataFrame(all_similar_pairs, columns=['Page1_Index', 'Page2_Index', 'Similarity_Score'])
+
+
+def identify_similar_text_sequences(
     df_combined: pd.DataFrame,
-    similarity_threshold: float = 0.9,
-    min_word_count: int = 10,
+    similarity_threshold: float = 1,
+    min_word_count: int = 1,
     min_consecutive_pages: int = 1,
-    greedy_match: bool = False,
-    combine_pages:bool=True,
-    output_folder: str = OUTPUT_FOLDER,
+    greedy_match: bool = True,
+    combine_pages: bool = False,
+    inter_file_only: bool = False,
+    do_text_clean:bool = True,
+    output_folder: str = "output/",
     progress=Progress(track_tqdm=True)
 ) -> Tuple[pd.DataFrame, List[str], pd.DataFrame]:
     """
-    Identifies similar pages with three possible strategies:
-    1. Single Page: If greedy_match=False and min_consecutive_pages=1.
-    2. Fixed-Length Subdocument: If greedy_match=False and min_consecutive_pages > 1.
-    3. Greedy Consecutive Match: If greedy_match=True.
+    Identifies similar pages. Uses a highly optimized path for inter_file_only=True.
     """
-
-    output_paths = []
     progress(0.1, desc="Processing and filtering text")
-    df = process_data(df_combined, 'text')
+
+    if do_text_clean:
+        df = clean_and_stem_text_series(df_combined, 'text') # Will produce the column 'text_clean'
+    else:
+        df = df_combined.copy()
+        df['text_clean'] = df['text'].str.lower().str.replace(r'[^\w\s]', '', regex=True)
+
     df['word_count'] = df['text_clean'].str.split().str.len().fillna(0)
+
     original_row_count = len(df)
     df_filtered = df[df['word_count'] >= min_word_count].copy()
     df_filtered.reset_index(drop=True, inplace=True)
-    
     print(f"Filtered out {original_row_count - len(df_filtered)} pages with fewer than {min_word_count} words.")
-
     if len(df_filtered) < 2:
         return pd.DataFrame(), [], df_combined
-        
-    vectorizer = TfidfVectorizer()
-    tfidf_matrix = vectorizer.fit_transform(df_filtered['text_clean'])
-
-    progress(0.3, desc="Calculating text similarity")
-    similarity_matrix = cosine_similarity(tfidf_matrix, dense_output=False)
-    coo_matrix = similarity_matrix.tocoo()
-    
-    # Create a DataFrame of all individual page pairs above the threshold.
-    # This is the base for all three matching strategies.
-    similar_pages = [
-        (r, c, v) for r, c, v in zip(coo_matrix.row, coo_matrix.col, coo_matrix.data)
-        if r < c and v >= similarity_threshold
-    ]
-
-    if not similar_pages:
-        return pd.DataFrame(), [], df_combined
-    
-    base_similarity_df = pd.DataFrame(similar_pages, columns=['Page1_Index', 'Page2_Index', 'Similarity_Score'])
 
-    progress(0.6, desc="Aggregating results based on matching strategy")
+    vectorizer = TfidfVectorizer()
     
-    if greedy_match:
-        print("Finding matches using greedy consecutive strategy.")
-        
-        # A set of pairs for fast lookups of (page1_idx, page2_idx)
-        valid_pairs_set = set(zip(base_similarity_df['Page1_Index'], base_similarity_df['Page2_Index']))
-        
-        # Keep track of indices that have been used in a sequence
-        consumed_indices_1 = set()
-        consumed_indices_2 = set()
-        
-        all_sequences = []
-
-        # Iterate through all potential starting pairs, sorted for consistent results
-        sorted_pairs = base_similarity_df.sort_values(['Page1_Index', 'Page2_Index'])
-
-        for _, row in sorted_pairs.iterrows():
-            start_idx1, start_idx2 = int(row['Page1_Index']), int(row['Page2_Index'])
+    # Similarity calculated differently if comparing between files only (inter_file_only==True), or within the same file
+    if inter_file_only:
+        # Use the new, highly efficient helper function.
+        base_similarity_df = _calculate_inter_file_similarity(df_filtered, vectorizer, similarity_threshold, progress)
+        if base_similarity_df.empty:
+            return pd.DataFrame(), [], df_combined
             
-            # If this pair has already been consumed by a previous sequence, skip it
-            if start_idx1 in consumed_indices_1 or start_idx2 in consumed_indices_2:
-                continue
+    else:
+        # Use the original, simpler path for all-to-all comparisons (including intra-file).
+        print("Standard Path: Calculating all-to-all similarity.")
+        progress(0.2, desc="Vectorizing text...")
+        tfidf_matrix = vectorizer.fit_transform(df_filtered['text_clean'])
 
-            # This is a new sequence, start expanding it
-            current_sequence = [(start_idx1, start_idx2)]
-            k = 1
-            while True:
-                next_idx1 = start_idx1 + k
-                next_idx2 = start_idx2 + k
-                
-                # Check if the next pair in the sequence is a valid match
-                if (next_idx1, next_idx2) in valid_pairs_set and \
-                   next_idx1 not in consumed_indices_1 and \
-                   next_idx2 not in consumed_indices_2:
-                    current_sequence.append((next_idx1, next_idx2))
-                    k += 1
-                else:
-                    # The sequence has ended
-                    break
-            
-            # Record the found sequence and mark all its pages as consumed
-            sequence_indices_1 = [p[0] for p in current_sequence]
-            sequence_indices_2 = [p[1] for p in current_sequence]
-            
-            all_sequences.append({
-                'Page1_Start_Index': sequence_indices_1[0], 'Page1_End_Index': sequence_indices_1[-1],
-                'Page2_Start_Index': sequence_indices_2[0], 'Page2_End_Index': sequence_indices_2[-1],
-                'Match_Length': len(current_sequence)
-            })
+        progress(0.3, desc="Calculating similarity matrix...")
+        similarity_matrix = cosine_similarity(tfidf_matrix, dense_output=False)
+        coo_matrix = similarity_matrix.tocoo()
 
-            consumed_indices_1.update(sequence_indices_1)
-            consumed_indices_2.update(sequence_indices_2)
+        similar_pages = [
+            (r, c, v) for r, c, v in zip(coo_matrix.row, coo_matrix.col, coo_matrix.data)
+            if r < c and v >= similarity_threshold
+        ]
 
-        if not all_sequences:
+        if not similar_pages:
             return pd.DataFrame(), [], df_combined
+        
+        base_similarity_df = pd.DataFrame(similar_pages, columns=['Page1_Index', 'Page2_Index', 'Similarity_Score'])
+
+    progress(0.6, desc="Aggregating results based on matching strategy")
 
-        subdocument_df = pd.DataFrame(all_sequences)
+    if greedy_match or min_consecutive_pages > 1:
+        print("Finding all consecutive page matches of minimum length:", min_consecutive_pages)
+        
+        # Sort the dataframe to ensure consecutive pages are adjacent
+        similarity_df = base_similarity_df #.sort_values(['Page1_Index', 'Page2_Index']).copy()
 
-    elif min_consecutive_pages > 1:
-        # --- STRATEGY 2: Fixed-Length Subdocument Matching ---
-        print(f"Finding consecutive page matches (min_consecutive_pages > 1)")
-        similarity_df = base_similarity_df.copy()
-        similarity_df.sort_values(['Page1_Index', 'Page2_Index'], inplace=True)
+        # A new sequence starts if the difference from the previous row is not (1, 1)
+        # is_consecutive will be True if a row continues the sequence, False if it's a new one.
         is_consecutive = (similarity_df['Page1_Index'].diff() == 1) & (similarity_df['Page2_Index'].diff() == 1)
+
+        # Use cumsum() on the inverted boolean series to create a unique ID for each block.
+        # Every time a 'False' appears (a new block starts), the sum increases.
         block_id = is_consecutive.eq(False).cumsum()
+
+        # Group by this block ID
         grouped = similarity_df.groupby(block_id)
+
+        # Aggregate each group to get the start, end, and length of the match
         agg_results = grouped.agg(
-            Page1_Start_Index=('Page1_Index', 'first'), Page2_Start_Index=('Page2_Index', 'first'),
-            Page1_End_Index=('Page1_Index', 'last'), Page2_End_Index=('Page2_Index', 'last'),
-            Match_Length=('Page1_Index', 'size'), Avg_Similarity=('Similarity_Score', 'mean')
+            Page1_Start_Index=('Page1_Index', 'first'),
+            Page2_Start_Index=('Page2_Index', 'first'),
+            Page1_End_Index=('Page1_Index', 'last'),
+            Page2_End_Index=('Page2_Index', 'last'),
+            Match_Length=('Page1_Index', 'size'),
+            Avg_Similarity=('Similarity_Score', 'mean')
         ).reset_index(drop=True)
-        subdocument_df = agg_results[agg_results['Match_Length'] >= min_consecutive_pages].copy()
-        if subdocument_df.empty: return pd.DataFrame(), [], df_combined
 
+        # If greedy_match=True, we keep all matches. If min_consecutive_pages > 1, we filter.
+        if greedy_match and min_consecutive_pages <= 1:
+            subdocument_df = agg_results
+        else:
+             # This handles the case for min_consecutive_pages > 1
+            subdocument_df = agg_results[agg_results['Match_Length'] >= min_consecutive_pages].copy()
+
+        if subdocument_df.empty:
+            gr.Info("No matches found")
+            return pd.DataFrame(), [], df_combined
+        
+        final_df = map_metadata_subdocument(subdocument_df, df_filtered)
     else:
-        # --- STRATEGY 1: Single Page Matching ---
-        print(f"Finding single page matches (min_consecutive_pages=1)")
+        print(f"Finding single page matches, not greedy (min_consecutive_pages=1)")
+        # This part of your code would handle the non-sequential case
         final_df = map_metadata_single_page(base_similarity_df, df_filtered)
-        # The rest of the logic (saving files) is handled after this if/else block
-        pass # The final_df is already prepared
+        subdocument_df = final_df # To align variable names for saving
+
+        if subdocument_df.empty:
+            gr.Info("No matches found")
+            return pd.DataFrame(), [], df_combined
 
-    # --- Map metadata and format output ---
-    # This block now handles the output for both subdocument strategies (2 and 3)
-    if greedy_match or min_consecutive_pages > 1:
-        final_df = map_metadata_subdocument(subdocument_df, df_filtered)
-    
     progress(0.8, desc="Saving output files")
     
     output_paths = save_results_and_redaction_lists(final_df, output_folder, combine_pages)
@@ -405,18 +901,16 @@ def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:
     Wrapper function updated to include the 'greedy_match' boolean.
     """
     if not files:
-        gr.Warning("Please upload files to analyze.")
-        return None, None, None
+        raise Warning("Please upload files to analyse.")
         
     progress(0, desc="Combining input files...")
-    df_combined, _ = combine_ocr_output_text(files, combine_pages=combine_pages)
+    df_combined, _, full_out_ocr_df = combine_ocr_output_text(files, combine_pages=combine_pages)
 
     if df_combined.empty:
-        gr.Warning("No data found in the uploaded files.")
-        return None, None, None
+        raise Warning("No data found in the uploaded files.")
 
     # Call the main analysis function with the new parameter
-    results_df, output_paths, full_df = identify_similar_pages(
+    results_df, output_paths, full_df = identify_similar_text_sequences(
         df_combined=df_combined,
         similarity_threshold=threshold,
         min_word_count=min_words,
@@ -428,7 +922,6 @@ def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:
 
     # Clip text to first 200 characters
     full_df['text'] = full_df['text'].str[:preview_length]
-
     # Preprocess full_data (without preview text) for fast access (run once)
     full_data_by_file = {
     file: df.sort_values('page').set_index('page')
@@ -438,7 +931,7 @@ def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:
     if results_df.empty:
         gr.Info(f"No duplicate pages found, no results returned.")
     
-    return results_df, output_paths, full_data_by_file # full_df, 
+    return results_df, output_paths, full_data_by_file
 
 def show_page_previews(full_data_by_file: dict, results_df: pd.DataFrame, evt: gr.SelectData, preview_length:int=500):
     """
@@ -523,10 +1016,29 @@ def add_new_annotations_to_existing_page_annotations(
 
     return all_annotations, newly_added_annotation_group
 
-def apply_whole_page_redactions_from_list(duplicate_page_numbers_df: pd.DataFrame, doc_file_name_with_extension_textbox: str, review_file_state: pd.DataFrame, duplicate_output_paths: list[str], pymupdf_doc: object, page_sizes: list[dict], all_existing_annotations: list[dict], combine_pages:bool=True, new_annotations_with_bounding_boxes:List[dict]=[]):
+def apply_whole_page_redactions_from_list(duplicate_page_numbers_df: pd.DataFrame, doc_file_name_with_extension_textbox: str, review_file_state: pd.DataFrame, duplicate_output_paths: list[str], pymupdf_doc: object, page_sizes: list[dict], all_existing_annotations: list[dict], combine_pages:bool=True, new_annotations_with_bounding_boxes:List[dict]=list()):
     '''
-    Take a list of suggested whole pages to redact and apply it to review file data.
+    This function applies redactions to whole pages based on a provided list of duplicate page numbers. It supports two modes of operation: combining pages and not combining pages. When combining pages is enabled, it attempts to identify duplicate pages across different files and applies redactions accordingly. If combining pages is disabled, it relies on new annotations with bounding boxes to determine which pages to redact. The function utilises a PyMuPDF document object to manipulate the PDF file, and it also considers the sizes of pages to ensure accurate redaction application.
+
+    Args:
+        duplicate_page_numbers_df (pd.DataFrame): A DataFrame containing page numbers identified as duplicates.
+        doc_file_name_with_extension_textbox (str): The name of the document file with its extension.
+        review_file_state (pd.DataFrame): The current state of the review file.
+        duplicate_output_paths (list[str]): A list of paths to files containing duplicate page information.
+        pymupdf_doc (object): A PyMuPDF document object representing the PDF file.
+        page_sizes (list[dict]): A list of dictionaries containing page size information.
+        all_existing_annotations (list[dict]): A list of all existing annotations in the document.
+        combine_pages (bool, optional): A flag indicating whether to combine pages for redaction. Defaults to True.
+        new_annotations_with_bounding_boxes (List[dict], optional): A list of new annotations with bounding boxes. Defaults to an empty list.
     '''
+    if all_existing_annotations is None:
+        all_existing_annotations = []
+
+    if new_annotations_with_bounding_boxes is None:
+        new_annotations_with_bounding_boxes = []
+
+    print("new_annotations_with_bounding_boxes:", new_annotations_with_bounding_boxes)
+
     all_annotations = all_existing_annotations.copy()
 
     if not pymupdf_doc:
@@ -659,34 +1171,27 @@ def apply_whole_page_redactions_from_list(duplicate_page_numbers_df: pd.DataFram
     review_file_out = review_file_out.sort_values(by=["page", "ymin", "xmin"]).reset_index(drop=True)
     review_file_out = review_file_out.drop_duplicates(subset=['page', 'label', 'text', 'id'], keep='first')
     
-    out_message = "Successfully created whole page redactions."
+    out_message = "Successfully created duplicate text redactions."
     print(out_message)
     gr.Info(out_message)
 
     return review_file_out, all_annotations
 
-# --- 1. Helper Function to Parse the Combined Page/Line ID ---
 def _parse_page_line_id(combined_id: int) -> Tuple[int, int]:
-    """
-    Parses a combined page and line number ID into a (page, line) tuple.
-    Assumes the ID is a 10-digit number where the first 5 are the page
-    and the last 5 are the line number.
-    
-    Example: 100027 -> (1, 27)
-             200005 -> (2, 5)
-    """
-    # zfill ensures the string is padded with leading zeros to 10 characters
-    s_id = str(combined_id).zfill(10)
-    page = int(s_id[:5])
-    line = int(s_id[5:])
+    """Parses a combined ID using modular arithmetic."""
+    if int(combined_id) < ID_MULTIPLIER:
+        # Handle cases where page is 0 (or just an edge case)
+        return 0, combined_id
+        
+    page = combined_id // ID_MULTIPLIER
+    line = combined_id % ID_MULTIPLIER
     return page, line
 
 def create_annotation_objects_from_duplicates(
     duplicates_df: pd.DataFrame, 
     ocr_results_df: pd.DataFrame,
     page_sizes: List[Dict],
-    combine_pages:bool=False
-) -> List[Dict]:
+    combine_pages:bool=False) -> List[Dict]:
     """
     Creates structured annotation objects from duplicate line ranges, mapping
     page numbers to image paths.
@@ -702,8 +1207,12 @@ def create_annotation_objects_from_duplicates(
     """
     final_output = []
 
+    if duplicates_df.empty:
+        raise Warning("No duplicates found")
+    if ocr_results_df.empty:
+        raise Warning("No OCR results found for file under review. Please upload relevant OCR_output file for the PDF file on the review tab.")
+
     if combine_pages == False:
-        # --- NEW: Create an efficient lookup map from page number to image path ---
         page_to_image_map = {item['page']: item['image_path'] for item in page_sizes}
 
         # Prepare OCR Data: Add a line number column if it doesn't exist

tools/helper_functions.py

@@ -22,7 +22,7 @@ def reset_state_vars():
             show_share_button=False,
             show_remove_button=False,
             interactive=False
-        ), [], [], pd.DataFrame(), pd.DataFrame(), [], [], "", False, 0
+        ), [], [], pd.DataFrame(), pd.DataFrame(), [], [], "", False, 0, []
 
 def reset_ocr_results_state():
     return pd.DataFrame(), pd.DataFrame(), []
@@ -573,7 +573,10 @@ def reset_base_dataframe(df:pd.DataFrame):
     return df
 
 def reset_ocr_base_dataframe(df:pd.DataFrame):
-    return df.iloc[:, [0,1]]
+    if df.empty:
+        return pd.DataFrame()
+    else:
+        return df.loc[:, ["page", "text"]]
 
 def reset_ocr_with_words_base_dataframe(df:pd.DataFrame, page_entity_dropdown_redaction_value:str):
     

tools/redaction_review.py

@@ -404,6 +404,68 @@ def _generate_unique_ids(
             
     return list(newly_generated_ids)
 
+def _merge_horizontally_adjacent_boxes(
+    df: pd.DataFrame, 
+    x_merge_threshold: int = 0.02
+) -> pd.DataFrame:
+    """
+    Merges horizontally adjacent bounding boxes within the same line.
+
+    Args:
+        df (pd.DataFrame): DataFrame containing annotation boxes with columns
+                           like 'page', 'line', 'xmin', 'xmax', etc.
+        x_merge_threshold (int): The maximum pixel gap on the x-axis to
+                                 consider two boxes as adjacent.
+
+    Returns:
+        pd.DataFrame: A new DataFrame with adjacent boxes merged.
+    """
+    if df.empty:
+        return df
+
+    # 1. Sort values to ensure we are comparing adjacent boxes
+    df_sorted = df.sort_values(by=['page', 'line', 'xmin']).copy()
+    
+    # 2. Identify groups of boxes to merge using shift() and cumsum()
+    # Get properties of the 'previous' box in the sorted list
+    prev_xmax = df_sorted['xmax'].shift(1)
+    prev_page = df_sorted['page'].shift(1)
+    prev_line = df_sorted['line'].shift(1)
+    
+    # A box should be merged with the previous one if it's on the same page/line
+    # and the horizontal gap is within the threshold.
+    is_adjacent = (
+        (df_sorted['page'] == prev_page) &
+        (df_sorted['line'] == prev_line) &
+        (df_sorted['xmin'] - prev_xmax <= x_merge_threshold)
+    )
+    
+    # A new group starts wherever a box is NOT adjacent to the previous one.
+    # cumsum() on this boolean series creates a unique ID for each group.
+    df_sorted['merge_group'] = (~is_adjacent).cumsum()
+    
+    # 3. Aggregate each group into a single bounding box
+    # Define how to aggregate each column
+    agg_funcs = {
+        'xmin': 'min',
+        'ymin': 'min', # To get the highest point of the combined box
+        'xmax': 'max',
+        'ymax': 'max', # To get the lowest point of the combined box
+        'text': lambda s: ' '.join(s.astype(str)), # Join the text
+        # Carry over the first value for columns that are constant within a group
+        'page': 'first',
+        'line': 'first',
+        'image': 'first',
+        'label': 'first',
+        'color': 'first',
+    }
+    
+    merged_df = df_sorted.groupby('merge_group').agg(agg_funcs).reset_index(drop=True)
+    
+    print(f"Merged {len(df)} annotations into {len(merged_df)}.")
+    
+    return merged_df
+
 def create_annotation_objects_from_filtered_ocr_results_with_words(
     filtered_ocr_results_with_words_df: pd.DataFrame, 
     ocr_results_with_words_df_base: pd.DataFrame,
@@ -411,54 +473,45 @@ def create_annotation_objects_from_filtered_ocr_results_with_words(
     existing_annotations_df: pd.DataFrame,
     existing_annotations_list: List[Dict],
     existing_recogniser_entity_df: pd.DataFrame,
-    progress=gr.Progress(track_tqdm=True)
+    progress = gr.Progress(track_tqdm=True)
 ) -> Tuple[List[Dict], List[Dict], pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     """
-    Creates and merges new annotations, using custom ID logic.
+    This function processes filtered OCR results with words to create new annotation objects. It merges these new annotations with existing ones, ensuring that horizontally adjacent boxes are combined for cleaner redactions. The function also updates the existing recogniser entity DataFrame and returns the updated annotations in both DataFrame and list-of-dicts formats.
+
+    Args:
+        filtered_ocr_results_with_words_df (pd.DataFrame): A DataFrame containing filtered OCR results with words.
+        ocr_results_with_words_df_base (pd.DataFrame): The base DataFrame of OCR results with words.
+        page_sizes (List[Dict]): A list of dictionaries containing page sizes.
+        existing_annotations_df (pd.DataFrame): A DataFrame of existing annotations.
+        existing_annotations_list (List[Dict]): A list of dictionaries representing existing annotations.
+        existing_recogniser_entity_df (pd.DataFrame): A DataFrame of existing recogniser entities.
+        progress (gr.Progress, optional): A progress tracker. Defaults to gr.Progress(track_tqdm=True).
+
+    Returns:
+        Tuple[List[Dict], List[Dict], pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: A tuple containing the updated annotations list, updated existing annotations list, updated annotations DataFrame, updated existing annotations DataFrame, updated recogniser entity DataFrame, and the original existing recogniser entity DataFrame.
     """
 
+    progress(0.2, "Identifying new redactions to add")  
     print("Identifying new redactions to add")
-    progress(0.1, "Identifying new redactions to add")
     if filtered_ocr_results_with_words_df.empty:
         print("No new annotations to add.")
         updated_annotations_df = existing_annotations_df.copy()
     else:
-        # join_keys = ['page', 'line', 'word_text', 'word_x0']
-        # new_annotations_df = pd.merge(
-        #     ocr_results_with_words_df_base,
-        #     filtered_ocr_results_with_words_df[join_keys],
-        #     on=join_keys,
-        #     how='inner'
-        # )
-
+        # Assuming index relationship holds for fast lookup
         filtered_ocr_results_with_words_df.index = filtered_ocr_results_with_words_df["index"]
-
         new_annotations_df = ocr_results_with_words_df_base.loc[filtered_ocr_results_with_words_df.index].copy()
 
         if new_annotations_df.empty:
              print("No new annotations to add.")
              updated_annotations_df = existing_annotations_df.copy()
         else:
-            # --- Custom ID Generation ---
-            progress(0.2, "Creating new redaction IDs")
-            # 1. Get all IDs that already exist to ensure we don't create duplicates.
-            existing_ids = set()
-            if 'id' in existing_annotations_df.columns:
-                existing_ids = set(existing_annotations_df['id'].dropna())
-            
-            # 2. Generate the exact number of new, unique IDs required.
-            num_new_ids = len(new_annotations_df)
-            new_id_list = _generate_unique_ids(num_new_ids, existing_ids)
-            
-            # 3. Assign the new IDs and other columns in a vectorized way.
             page_to_image_map = {item['page']: item['image_path'] for item in page_sizes}
             
-            progress(0.4, "Assigning new redaction details to dataframe")
+            # Prepare the initial new annotations DataFrame
             new_annotations_df = new_annotations_df.assign(
                 image=lambda df: df['page'].map(page_to_image_map),
                 label="Redaction",
-                color='(0, 0, 0)',
-                id=new_id_list  # Assign the pre-generated list of unique IDs
+                color='(0, 0, 0)'
             ).rename(columns={
                 'word_x0': 'xmin',
                 'word_y0': 'ymin',
@@ -466,6 +519,17 @@ def create_annotation_objects_from_filtered_ocr_results_with_words(
                 'word_y1': 'ymax',
                 'word_text': 'text'
             })
+
+            progress(0.3, "Checking for adjacent annotations to merge...")            
+            print("Checking for adjacent annotations to merge...")
+            new_annotations_df = _merge_horizontally_adjacent_boxes(new_annotations_df)
+
+            progress(0.4, "Creating new redaction IDs...")
+            print("Creating new redaction IDs...")
+            existing_ids = set(existing_annotations_df['id'].dropna()) if 'id' in existing_annotations_df.columns else set()
+            num_new_ids = len(new_annotations_df)
+            new_id_list = _generate_unique_ids(num_new_ids, existing_ids)
+            new_annotations_df['id'] = new_id_list
             
             annotation_cols = ['image', 'page', 'label', 'color', 'xmin', 'ymin', 'xmax', 'ymax', 'text', 'id']
             new_annotations_df = new_annotations_df[annotation_cols]
@@ -477,7 +541,7 @@ def create_annotation_objects_from_filtered_ocr_results_with_words(
             if existing_annotations_df.empty or not all(col in existing_annotations_df.columns for col in key_cols):
                 unique_new_df = new_annotations_df
             else:
-                # I'm not doing checks on this anymore
+                # I'm not doing checks against existing as it is too compute intensive in large documents
                 # merged = pd.merge(
                 #     new_annotations_df,
                 #     existing_annotations_df[key_cols].drop_duplicates(),
@@ -508,16 +572,39 @@ def create_annotation_objects_from_filtered_ocr_results_with_words(
         merged_df = pd.merge(all_pages_df[['image']], updated_annotations_df, on='image', how='left')
     else:
         merged_df = all_pages_df[['image']]
-        
+
+    # 1. Get the list of image paths in the exact order they appear in page_sizes.
+    #    all_pages_df was created from page_sizes, so it preserves this order.
+    image_order = all_pages_df['image'].tolist()
+
+    # 2. Convert the 'image' column to a special 'Categorical' type.
+    #    This tells pandas that this column has a custom, non-alphabetical order.
+    merged_df['image'] = pd.Categorical(merged_df['image'], categories=image_order, ordered=True)
+
+    # 3. Sort the DataFrame based on this new custom order.
+    merged_df = merged_df.sort_values('image')
+    
+    # --- NEW CODE END ---
+    
     final_annotations_list = []
     box_cols = ['label', 'color', 'xmin', 'ymin', 'xmax', 'ymax', 'text', 'id']
     
-    for image_path, group in progress.tqdm(merged_df.groupby('image'), desc="Adding redaction boxes to annotation object"):
+    # Now, when we group, we use `sort=False`. This tells groupby to respect the
+    # DataFrame's current order, which we have just manually set. This is slightly
+    # more efficient than letting it sort again.
+    for image_path, group in merged_df.groupby('image', sort=False):
+        # The progress.tqdm wrapper can be added back around the groupby object as you had it.
+        # for image_path, group in progress.tqdm(merged_df.groupby('image', sort=False), ...):
+        
+        # Check if the group has actual annotations. iloc[0] is safe because even pages
+        # without annotations will have one row with NaN values from the merge.
         if pd.isna(group.iloc[0].get('id')):
             boxes = []
         else:
             valid_box_cols = [col for col in box_cols if col in group.columns]
-            boxes = group[valid_box_cols].to_dict('records')
+            # We should also sort the boxes within a page for consistency (e.g., left-to-right)
+            sorted_group = group.sort_values(by=['ymin', 'xmin'])
+            boxes = sorted_group[valid_box_cols].to_dict('records')
             
         final_annotations_list.append({
             "image": image_path,