Added example data files. Greatly revised CLI redaction for redaction, deduplication, and AWS Textract batch calls. Various minor fixes and package updates.

.dockerignore

@@ -1,8 +1,4 @@
-*.csv
-*.pdf
 *.url
-*.jpg
-*.png
 *.ipynb
 *.pyc
 examples/*

.gitattributes

@@ -0,0 +1,7 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.xls filter=lfs diff=lfs merge=lfs -text
+*.xlsx filter=lfs diff=lfs merge=lfs -text
+*.docx filter=lfs diff=lfs merge=lfs -text
+*.doc filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,8 +1,4 @@
-*.csv
-*.pdf
 *.url
-*.jpg
-*.png
 *.ipynb
 *.pyc
 examples/*
@@ -17,8 +13,9 @@ build/*
 dist/*
 build_deps/*
 logs/*
+usage/*
+feedback/*
 config/*
-doc_redaction_amplify_app/*
 user_guide/*
 cdk/config/*
 cdk/cdk.out/*

app.py

@@ -2,7 +2,7 @@ import os
 import pandas as pd
 import gradio as gr
 from gradio_image_annotation import image_annotator
-from tools.config import DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX, DEFAULT_TABULAR_ANONYMISATION_STRATEGY, OUTPUT_FOLDER, INPUT_FOLDER, RUN_DIRECT_MODE, MAX_QUEUE_SIZE, DEFAULT_CONCURRENCY_LIMIT, MAX_FILE_SIZE, GRADIO_SERVER_PORT, ROOT_PATH, GET_DEFAULT_ALLOW_LIST, ALLOW_LIST_PATH, S3_ALLOW_LIST_PATH, FEEDBACK_LOGS_FOLDER, ACCESS_LOGS_FOLDER, USAGE_LOGS_FOLDER, DEFAULT_LANGUAGE, GET_COST_CODES, COST_CODES_PATH, S3_COST_CODES_PATH, ENFORCE_COST_CODES, DISPLAY_FILE_NAMES_IN_LOGS, SHOW_COSTS, RUN_AWS_FUNCTIONS, DOCUMENT_REDACTION_BUCKET, SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER, SESSION_OUTPUT_FOLDER, LOAD_PREVIOUS_TEXTRACT_JOBS_S3, TEXTRACT_JOBS_S3_LOC, TEXTRACT_JOBS_LOCAL_LOC, HOST_NAME, DEFAULT_COST_CODE, OUTPUT_COST_CODES_PATH, OUTPUT_ALLOW_LIST_PATH, COGNITO_AUTH, SAVE_LOGS_TO_CSV, SAVE_LOGS_TO_DYNAMODB, ACCESS_LOG_DYNAMODB_TABLE_NAME, DYNAMODB_ACCESS_LOG_HEADERS, CSV_ACCESS_LOG_HEADERS, FEEDBACK_LOG_DYNAMODB_TABLE_NAME, DYNAMODB_FEEDBACK_LOG_HEADERS, CSV_FEEDBACK_LOG_HEADERS, USAGE_LOG_DYNAMODB_TABLE_NAME, DYNAMODB_USAGE_LOG_HEADERS, CSV_USAGE_LOG_HEADERS, TEXTRACT_JOBS_S3_INPUT_LOC,  TEXTRACT_TEXT_EXTRACT_OPTION, NO_REDACTION_PII_OPTION, TEXT_EXTRACTION_MODELS, PII_DETECTION_MODELS, DEFAULT_TEXT_EXTRACTION_MODEL, DEFAULT_PII_DETECTION_MODEL, LOG_FILE_NAME, CHOSEN_COMPREHEND_ENTITIES, FULL_COMPREHEND_ENTITY_LIST, CHOSEN_REDACT_ENTITIES, FULL_ENTITY_LIST, FILE_INPUT_HEIGHT, TABULAR_PII_DETECTION_MODELS, USAGE_LOG_FILE_NAME, FEEDBACK_LOG_FILE_NAME, CONFIG_FOLDER, GRADIO_TEMP_DIR, MPLCONFIGDIR, S3_FEEDBACK_LOGS_FOLDER, S3_ACCESS_LOGS_FOLDER, S3_USAGE_LOGS_FOLDER, CHOSEN_LOCAL_OCR_MODEL, DEFAULT_LANGUAGE_FULL_NAME, SHOW_LANGUAGE_SELECTION, DO_INITIAL_TABULAR_DATA_CLEAN, DEFAULT_DUPLICATE_DETECTION_THRESHOLD, DIRECT_MODE_TASK, DIRECT_MODE_INPUT_FILE, DIRECT_MODE_OUTPUT_DIR, DIRECT_MODE_DUPLICATE_TYPE, LOCAL_PII_OPTION, TESSERACT_TEXT_EXTRACT_OPTION, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, DOCUMENT_REDACTION_BUCKET, IMAGES_DPI, PREPROCESS_LOCAL_OCR_IMAGES, COMPRESS_REDACTED_PDF, RETURN_PDF_END_OF_REDACTION, OUTPUT_DENY_LIST_PATH, OUTPUT_WHOLE_PAGE_REDACTION_LIST_PATH, DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX, DEFAULT_TABULAR_ANONYMISATION_STRATEGY, DEFAULT_FUZZY_SPELLING_MISTAKES_NUM, DEFAULT_PAGE_MIN, DEFAULT_PAGE_MAX, DEFAULT_EXCEL_SHEETS, USE_GREEDY_DUPLICATE_DETECTION, DEFAULT_MIN_CONSECUTIVE_PAGES, DEFAULT_COMBINE_PAGES, DEFAULT_MIN_WORD_COUNT, DEFAULT_TEXT_COLUMNS, DEFAULT_SEARCH_QUERY
+from tools.config import DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX, DEFAULT_TABULAR_ANONYMISATION_STRATEGY, OUTPUT_FOLDER, INPUT_FOLDER, RUN_DIRECT_MODE, MAX_QUEUE_SIZE, DEFAULT_CONCURRENCY_LIMIT, MAX_FILE_SIZE, GRADIO_SERVER_PORT, ROOT_PATH, GET_DEFAULT_ALLOW_LIST, ALLOW_LIST_PATH, S3_ALLOW_LIST_PATH, FEEDBACK_LOGS_FOLDER, ACCESS_LOGS_FOLDER, USAGE_LOGS_FOLDER, DEFAULT_LANGUAGE, GET_COST_CODES, COST_CODES_PATH, S3_COST_CODES_PATH, ENFORCE_COST_CODES, DISPLAY_FILE_NAMES_IN_LOGS, SHOW_COSTS, RUN_AWS_FUNCTIONS, DOCUMENT_REDACTION_BUCKET, SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER, SESSION_OUTPUT_FOLDER, LOAD_PREVIOUS_TEXTRACT_JOBS_S3, TEXTRACT_JOBS_S3_LOC, TEXTRACT_JOBS_LOCAL_LOC, HOST_NAME, DEFAULT_COST_CODE, OUTPUT_COST_CODES_PATH, ALLOW_LIST_PATH, COGNITO_AUTH, SAVE_LOGS_TO_CSV, SAVE_LOGS_TO_DYNAMODB, ACCESS_LOG_DYNAMODB_TABLE_NAME, DYNAMODB_ACCESS_LOG_HEADERS, CSV_ACCESS_LOG_HEADERS, FEEDBACK_LOG_DYNAMODB_TABLE_NAME, DYNAMODB_FEEDBACK_LOG_HEADERS, CSV_FEEDBACK_LOG_HEADERS, USAGE_LOG_DYNAMODB_TABLE_NAME, DYNAMODB_USAGE_LOG_HEADERS, CSV_USAGE_LOG_HEADERS, TEXTRACT_JOBS_S3_INPUT_LOC,  TEXTRACT_TEXT_EXTRACT_OPTION, NO_REDACTION_PII_OPTION, TEXT_EXTRACTION_MODELS, PII_DETECTION_MODELS, DEFAULT_TEXT_EXTRACTION_MODEL, DEFAULT_PII_DETECTION_MODEL, LOG_FILE_NAME, CHOSEN_COMPREHEND_ENTITIES, FULL_COMPREHEND_ENTITY_LIST, CHOSEN_REDACT_ENTITIES, FULL_ENTITY_LIST, FILE_INPUT_HEIGHT, TABULAR_PII_DETECTION_MODELS, USAGE_LOG_FILE_NAME, FEEDBACK_LOG_FILE_NAME, CONFIG_FOLDER, GRADIO_TEMP_DIR, MPLCONFIGDIR, S3_FEEDBACK_LOGS_FOLDER, S3_ACCESS_LOGS_FOLDER, S3_USAGE_LOGS_FOLDER, CHOSEN_LOCAL_OCR_MODEL, DEFAULT_LANGUAGE_FULL_NAME, SHOW_LANGUAGE_SELECTION, DO_INITIAL_TABULAR_DATA_CLEAN, DEFAULT_DUPLICATE_DETECTION_THRESHOLD, DIRECT_MODE_TASK, DIRECT_MODE_INPUT_FILE, DIRECT_MODE_OUTPUT_DIR, DIRECT_MODE_DUPLICATE_TYPE, DIRECT_MODE_DEFAULT_USER, LOCAL_PII_OPTION, TESSERACT_TEXT_EXTRACT_OPTION, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, DOCUMENT_REDACTION_BUCKET, IMAGES_DPI, PREPROCESS_LOCAL_OCR_IMAGES, COMPRESS_REDACTED_PDF, RETURN_PDF_END_OF_REDACTION, DENY_LIST_PATH, WHOLE_PAGE_REDACTION_LIST_PATH, DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX, DEFAULT_TABULAR_ANONYMISATION_STRATEGY, DEFAULT_FUZZY_SPELLING_MISTAKES_NUM, DEFAULT_PAGE_MIN, DEFAULT_PAGE_MAX, DEFAULT_EXCEL_SHEETS, USE_GREEDY_DUPLICATE_DETECTION, DEFAULT_MIN_CONSECUTIVE_PAGES, DEFAULT_COMBINE_PAGES, DEFAULT_MIN_WORD_COUNT, DEFAULT_TEXT_COLUMNS, DEFAULT_SEARCH_QUERY, REMOVE_DUPLICATE_ROWS
 from tools.helper_functions import put_columns_in_df, get_connection_params, reveal_feedback_buttons, custom_regex_load, reset_state_vars, load_in_default_allow_list, reset_review_vars, merge_csv_files, load_all_output_files, update_dataframe, check_for_existing_textract_file, load_in_default_cost_codes, enforce_cost_codes, calculate_aws_costs, calculate_time_taken, reset_base_dataframe, reset_ocr_base_dataframe, update_cost_code_dataframe_from_dropdown_select, check_for_relevant_ocr_output_with_words, reset_data_vars, reset_aws_call_vars, _get_env_list, ensure_folder_exists, reset_ocr_with_words_base_dataframe, update_language_dropdown, LANGUAGE_CHOICES, MAPPED_LANGUAGE_CHOICES
 from tools.aws_functions import download_file_from_s3, upload_log_file_to_s3
 from tools.file_redaction import choose_and_run_redactor
@@ -14,6 +14,7 @@ from tools.load_spacy_model_custom_recognisers import custom_entities
 from tools.custom_csvlogger import CSVLogger_custom
 from tools.find_duplicate_pages import run_duplicate_analysis, exclude_match, handle_selection_and_preview, apply_whole_page_redactions_from_list, create_annotation_objects_from_duplicates, run_full_search_and_analysis
 from tools.find_duplicate_tabular import update_tabular_column_choices, run_tabular_duplicate_detection, handle_tabular_row_selection, clean_tabular_duplicates
+import time
 from tools.textract_batch_call import analyse_document_with_textract_api, poll_whole_document_textract_analysis_progress_and_download, load_in_textract_job_details, check_for_provided_job_id, check_textract_outputs_exist, replace_existing_pdf_input_for_whole_document_outputs
 
 # Suppress downcasting warnings
@@ -48,6 +49,8 @@ if USE_GREEDY_DUPLICATE_DETECTION == "True": USE_GREEDY_DUPLICATE_DETECTION = Tr
 else: USE_GREEDY_DUPLICATE_DETECTION = False
 if DEFAULT_COMBINE_PAGES == "True": DEFAULT_COMBINE_PAGES = True 
 else: DEFAULT_COMBINE_PAGES = False
+if REMOVE_DUPLICATE_ROWS == "True": REMOVE_DUPLICATE_ROWS = True 
+else: REMOVE_DUPLICATE_ROWS = False
 
 if CSV_ACCESS_LOG_HEADERS: CSV_ACCESS_LOG_HEADERS = _get_env_list(CSV_ACCESS_LOG_HEADERS)
 if CSV_FEEDBACK_LOG_HEADERS: CSV_FEEDBACK_LOG_HEADERS = _get_env_list(CSV_FEEDBACK_LOG_HEADERS)
@@ -73,6 +76,58 @@ FULL_COMPREHEND_ENTITY_LIST.extend(custom_entities)
 
 FILE_INPUT_HEIGHT = int(FILE_INPUT_HEIGHT)
 
+# Wrapper functions to add timing to deduplication functions
+def run_duplicate_analysis_with_timing(files, threshold, min_words, min_consecutive, greedy_match, combine_pages, output_folder):
+    """
+    Wrapper for run_duplicate_analysis that adds timing and returns time taken.
+    """
+    start_time = time.time()
+    results_df, output_paths, full_data_by_file = run_duplicate_analysis(
+        files=files,
+        threshold=threshold,
+        min_words=min_words,
+        min_consecutive=min_consecutive,
+        greedy_match=greedy_match,
+        combine_pages=combine_pages,
+        output_folder=output_folder
+    )
+    end_time = time.time()
+    processing_time = end_time - start_time
+    
+    # Store the time taken in a global variable for logging
+    global duplicate_analysis_time_taken
+    duplicate_analysis_time_taken = processing_time
+    
+    return results_df, output_paths, full_data_by_file
+
+def run_tabular_duplicate_detection_with_timing(files, threshold, min_words, text_columns, output_folder, do_initial_clean_dup, in_excel_tabular_sheets, remove_duplicate_rows):
+    """
+    Wrapper for run_tabular_duplicate_detection that adds timing and returns time taken.
+    """
+    start_time = time.time()
+    results_df, output_paths, file_choices = run_tabular_duplicate_detection(
+        files=files,
+        threshold=threshold,
+        min_words=min_words,
+        text_columns=text_columns,
+        output_folder=output_folder,
+        do_initial_clean_dup=do_initial_clean_dup,
+        in_excel_tabular_sheets=in_excel_tabular_sheets,
+        remove_duplicate_rows=remove_duplicate_rows
+    )
+    end_time = time.time()
+    processing_time = end_time - start_time
+    
+    # Store the time taken in a global variable for logging
+    global tabular_duplicate_analysis_time_taken
+    tabular_duplicate_analysis_time_taken = processing_time
+    
+    return results_df, output_paths, file_choices
+
+# Initialize global variables for timing
+duplicate_analysis_time_taken = 0.0
+tabular_duplicate_analysis_time_taken = 0.0
+
 # Create the gradio interface
 app = gr.Blocks(theme = gr.themes.Default(primary_hue="blue"), fill_width=True) #gr.themes.Base()
 
@@ -185,7 +240,7 @@ with app:
     # S3 settings for default allow list load
     s3_default_bucket = gr.Textbox(label = "Default S3 bucket", value=DOCUMENT_REDACTION_BUCKET, visible=False)
     s3_default_allow_list_file = gr.Textbox(label = "Default allow list file", value=S3_ALLOW_LIST_PATH, visible=False)
-    default_allow_list_output_folder_location = gr.Textbox(label = "Output default allow list location", value=OUTPUT_ALLOW_LIST_PATH, visible=False)
+    default_allow_list_output_folder_location = gr.Textbox(label = "Output default allow list location", value=ALLOW_LIST_PATH, visible=False)
 
     s3_whole_document_textract_default_bucket = gr.Textbox(label = "Default Textract whole_document S3 bucket", value=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, visible=False)
     s3_whole_document_textract_input_subfolder = gr.Textbox(label = "Default Textract whole_document S3 input folder", value=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER, visible=False)
@@ -255,6 +310,7 @@ with app:
     textract_job_detail_df = gr.Dataframe(pd.DataFrame(columns=['job_id','file_name','job_type','signature_extraction','job_date_time']), label="Previous job details", visible=False, type="pandas", wrap=True)
     selected_job_id_row = gr.Dataframe(pd.DataFrame(columns=['job_id','file_name','job_type','signature_extraction','job_date_time']), label="Selected job id row", visible=False, type="pandas", wrap=True)
     is_a_textract_api_call = gr.Checkbox(value=False, label="is_this_a_textract_api_call", visible=False)
+    task_textbox = gr.Textbox(value="redact", label="task", visible=False)  # Track the task being performed
     job_output_textbox = gr.Textbox(value="", label="Textract call outputs", visible=False)
     job_input_textbox = gr.Textbox(value=TEXTRACT_JOBS_S3_INPUT_LOC, label="Textract call outputs", visible=False)
 
@@ -336,13 +392,13 @@ with app:
                             textract_job_output_file = gr.File(label="Textract job output files", height=100, visible=True)
                         with gr.Column():
                             job_current_status = gr.Textbox(value="", label="Analysis job current status", visible=True) 
-                            convert_textract_outputs_to_ocr_results = gr.Button("Convert Textract job outputs to OCR results", variant="secondary", visible=True)                           
+                            convert_textract_outputs_to_ocr_results = gr.Button("Convert Textract job outputs to OCR results", variant="secondary", visible=True)                  
 
             gr.Markdown("""If you only want to redact certain pages, or certain entities (e.g. just email addresses, or a custom list of terms), please go to the Redaction Settings tab.""")      
             document_redact_btn = gr.Button("Extract text and redact document", variant="primary", scale = 4)
         
-        with gr.Row(equal_height=True):
-            redaction_output_summary_textbox = gr.Textbox(label="Output summary", scale=1)
+        with gr.Row():
+            redaction_output_summary_textbox = gr.Textbox(label="Output summary", scale=1, lines=4)
             output_file = gr.File(label="Output files", scale = 2)#, height=FILE_INPUT_HEIGHT)
             latest_file_completed_num = gr.Number(value=0, label="Number of documents redacted", interactive=False, visible=False)
 
@@ -558,13 +614,13 @@ with app:
 
             with gr.Accordion("Anonymisation output format - by default will replace PII with a blank space", open = False):
                 with gr.Row():
-                    anon_strat = gr.Radio(choices=["replace with 'REDACTED'", "replace with <ENTITY_NAME>", "redact completely", "hash", "mask"], label="Select an anonymisation method.", value = DEFAULT_TABULAR_ANONYMISATION_STRATEGY) # , "encrypt", "fake_first_name" are also available, but are not currently included as not that useful in current form
+                    anon_strategy = gr.Radio(choices=["replace with 'REDACTED'", "replace with <ENTITY_NAME>", "redact completely", "hash", "mask"], label="Select an anonymisation method.", value = DEFAULT_TABULAR_ANONYMISATION_STRATEGY) # , "encrypt", "fake_first_name" are also available, but are not currently included as not that useful in current form
                     do_initial_clean = gr.Checkbox(label="Do initial clean of text (remove URLs, HTML tags, and non-ASCII characters)", value=DO_INITIAL_TABULAR_DATA_CLEAN)
             
             tabular_data_redact_btn = gr.Button("Redact text/data files", variant="primary")
             
-            with gr.Row(equal_height=True):
-                text_output_summary = gr.Textbox(label="Output result")
+            with gr.Row():
+                text_output_summary = gr.Textbox(label="Output result", lines=4)
                 text_output_file = gr.File(label="Output files")
                 text_tabular_files_done = gr.Number(value=0, label="Number of tabular files redacted", interactive=False, visible=False)
 
@@ -574,21 +630,21 @@ with app:
         # TABULAR DUPLICATE DETECTION TAB
         ###
         with gr.Accordion(label="Find duplicate cells in tabular data", open=False):
-            gr.Markdown("""Find duplicate cells or rows in CSV and Excel files. This tool analyzes text content across all columns to identify similar or identical entries that may be duplicates. You can review the results and choose to remove duplicate rows from your files.""")
+            gr.Markdown("""Find duplicate cells or rows in CSV, Excel, or Parquet files. This tool analyzes text content across all columns to identify similar or identical entries that may be duplicates. You can review the results and choose to remove duplicate rows from your files.""")
             
             with gr.Accordion("Step 1: Upload files and configure analysis", open=True):
                 in_tabular_duplicate_files = gr.File(
-                    label="Upload CSV or Excel files to find duplicate cells/rows",
+                    label="Upload CSV, Excel, or Parquet files to find duplicate cells/rows. Note that the app will remove duplicates from later cells/files that are found in earlier cells/files and not vice versa.",
                     file_count="multiple", 
                     file_types=['.csv', '.xlsx', '.xls', '.parquet'],
                     height=FILE_INPUT_HEIGHT
                 )
                 
-                with gr.Row():
+                with gr.Row(equal_height=True):
                     tabular_duplicate_threshold = gr.Number(
                         value=DEFAULT_DUPLICATE_DETECTION_THRESHOLD, 
                         label="Similarity threshold", 
-                        info="Score (0-1) to consider cells a match. Higher values = more strict matching."
+                        info="Score (0-1) to consider cells a match. 1 = perfect match."
                     )
                     tabular_min_word_count = gr.Number(
                         value=DEFAULT_MIN_WORD_COUNT, 
@@ -596,13 +652,17 @@ with app:
                         info="Cells with fewer words than this are ignored."
                     )
                     do_initial_clean_dup = gr.Checkbox(label="Do initial clean of text (remove URLs, HTML tags, and non-ASCII characters)", value=DO_INITIAL_TABULAR_DATA_CLEAN)
+                    remove_duplicate_rows = gr.Checkbox(label="Remove duplicate rows from deduplicated files", value=REMOVE_DUPLICATE_ROWS)
                 
-                tabular_text_columns = gr.Dropdown(
-                    choices=DEFAULT_TEXT_COLUMNS,
-                    multiselect=True,
-                    label="Select specific columns to analyse (leave empty to analyse all text columns)",
-                    info="If no columns selected, all text columns will be analyzed"
-                )
+                with gr.Row():
+                    in_excel_tabular_sheets = gr.Dropdown(choices=list(), multiselect = True, label="Select Excel sheet names that you want to deduplicate (showing sheets present across all Excel files).", visible=True, allow_custom_value=True)
+
+                    tabular_text_columns = gr.Dropdown(
+                        choices=DEFAULT_TEXT_COLUMNS,
+                        multiselect=True,
+                        label="Select specific columns to analyse (leave empty to analyse all text columns simultaneously - i.e. all text is joined together)",
+                        info="If no columns selected, all text columns will combined together and analysed"
+                    )
                 
                 find_tabular_duplicates_btn = gr.Button(
                     value="Find duplicate cells/rows", 
@@ -621,7 +681,7 @@ with app:
                     show_copy_button=True
                 )
                 
-                with gr.Row():
+                with gr.Row(equal_height=True):
                     tabular_selected_row_index = gr.Number(value=None, visible=False)
                     tabular_text1_preview = gr.Textbox(
                         label="Text from File 1", 
@@ -775,11 +835,11 @@ with app:
     document_redact_btn.click(fn = reset_state_vars, outputs=[all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, textract_metadata_textbox, annotator, output_file_list_state, log_files_output_list_state, recogniser_entity_dataframe, recogniser_entity_dataframe_base, pdf_doc_state, duplication_file_path_outputs_list_state, redaction_output_summary_textbox, is_a_textract_api_call, textract_query_number, all_page_line_level_ocr_results_with_words]).\
         success(fn= enforce_cost_codes, inputs=[enforce_cost_code_textbox, cost_code_choice_drop, cost_code_dataframe_base]).\
         success(fn= choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_entities, in_redact_comprehend_entities, text_extract_method_radio, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_num, redaction_output_summary_textbox, output_file_list_state, log_files_output_list_state, first_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_df, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, chosen_local_model_textbox, chosen_language_drop],
-                    outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, input_pdf_for_review, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state], api_name="redact_doc", show_progress_on=[redaction_output_summary_textbox])
+                    outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, input_pdf_for_review, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state, task_textbox], api_name="redact_doc", show_progress_on=[redaction_output_summary_textbox])
         
     # If a file has been completed, the function will continue onto the next document
     latest_file_completed_num.change(fn = choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_entities, in_redact_comprehend_entities, text_extract_method_radio, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_num, redaction_output_summary_textbox, output_file_list_state, log_files_output_list_state, second_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_df, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, chosen_local_model_textbox, chosen_language_drop],
-                    outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, input_pdf_for_review, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state], show_progress_on=[redaction_output_summary_textbox]).\
+                    outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, input_pdf_for_review, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state, task_textbox], show_progress_on=[redaction_output_summary_textbox]).\
                     success(fn=update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_df, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, page_sizes, all_image_annotations_state], show_progress_on=[annotator]).\
                     success(fn=check_for_existing_textract_file, inputs=[doc_file_name_no_extension_textbox, output_folder_textbox], outputs=[textract_output_found_checkbox]).\
                     success(fn=check_for_relevant_ocr_output_with_words, inputs=[doc_file_name_no_extension_textbox, text_extract_method_radio, output_folder_textbox], outputs=[relevant_ocr_output_with_words_found_checkbox]).\
@@ -791,7 +851,7 @@ with app:
     all_page_line_level_ocr_results_with_words_df_base.change(reset_ocr_with_words_base_dataframe, inputs=[all_page_line_level_ocr_results_with_words_df_base, page_entity_dropdown_redaction], outputs=[all_page_line_level_ocr_results_with_words_df, backup_all_page_line_level_ocr_results_with_words_df_base])
 
     # Send whole document to Textract for text extraction
-    send_document_to_textract_api_btn.click(analyse_document_with_textract_api, inputs=[prepared_pdf_state, s3_whole_document_textract_input_subfolder, s3_whole_document_textract_output_subfolder, textract_job_detail_df, s3_whole_document_textract_default_bucket, output_folder_textbox, handwrite_signature_checkbox, successful_textract_api_call_number, total_pdf_page_count], outputs=[job_output_textbox, job_id_textbox, job_type_dropdown, successful_textract_api_call_number, is_a_textract_api_call, textract_query_number]).\
+    send_document_to_textract_api_btn.click(analyse_document_with_textract_api, inputs=[prepared_pdf_state, s3_whole_document_textract_input_subfolder, s3_whole_document_textract_output_subfolder, textract_job_detail_df, s3_whole_document_textract_default_bucket, output_folder_textbox, handwrite_signature_checkbox, successful_textract_api_call_number, total_pdf_page_count], outputs=[job_output_textbox, job_id_textbox, job_type_dropdown, successful_textract_api_call_number, is_a_textract_api_call, textract_query_number, task_textbox]).\
         success(check_for_provided_job_id, inputs=[job_id_textbox]).\
         success(poll_whole_document_textract_analysis_progress_and_download, inputs=[job_id_textbox, job_type_dropdown, s3_whole_document_textract_output_subfolder, doc_file_name_no_extension_textbox, textract_job_detail_df, s3_whole_document_textract_default_bucket, output_folder_textbox, s3_whole_document_textract_logs_subfolder, local_whole_document_textract_logs_subfolder], outputs = [textract_job_output_file, job_current_status, textract_job_detail_df, doc_file_name_no_extension_textbox]).\
         success(fn=check_for_existing_textract_file, inputs=[doc_file_name_no_extension_textbox, output_folder_textbox], outputs=[textract_output_found_checkbox])
@@ -809,7 +869,7 @@ with app:
         success(fn= check_textract_outputs_exist, inputs=[textract_output_found_checkbox]).\
         success(fn = reset_state_vars, outputs=[all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, textract_metadata_textbox, annotator, output_file_list_state, log_files_output_list_state, recogniser_entity_dataframe, recogniser_entity_dataframe_base, pdf_doc_state, duplication_file_path_outputs_list_state, redaction_output_summary_textbox, is_a_textract_api_call, textract_query_number]).\
         success(fn= choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_entities, in_redact_comprehend_entities, textract_only_method_drop, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_num, redaction_output_summary_textbox, output_file_list_state, log_files_output_list_state, first_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, no_redaction_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_df, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, chosen_local_model_textbox, chosen_language_drop],
-                    outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, input_pdf_for_review, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state], show_progress_on=[redaction_output_summary_textbox]).\
+                    outputs=[redaction_output_summary_textbox, output_file, output_file_list_state, latest_file_completed_num, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_page_line_level_ocr_results_df_base, all_decision_process_table_state, comprehend_query_number, input_pdf_for_review, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_df, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number, latest_ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df_base, backup_review_state, task_textbox], show_progress_on=[redaction_output_summary_textbox]).\
                     success(fn=update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_df, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_entity_dropdown_redaction, page_sizes, all_image_annotations_state], show_progress_on=[annotator])
     
     ###
@@ -1021,10 +1081,10 @@ with app:
                   success(fn=get_input_file_names, inputs=[in_data_files], outputs=[data_file_name_no_extension_textbox, data_file_name_with_extension_textbox, data_full_file_name_textbox, data_file_name_textbox_list, total_pdf_page_count])
 
     tabular_data_redact_btn.click(reset_data_vars, outputs=[actual_time_taken_number, log_files_output_list_state, comprehend_query_number]).\
-    success(fn=anonymise_files_with_open_text, inputs=[in_data_files, in_text, anon_strat, in_colnames,  in_redact_entities, in_allow_list_state, text_tabular_files_done, text_output_summary, text_output_file_list_state, log_files_output_list_state, in_excel_sheets, first_loop_state, output_folder_textbox, in_deny_list_state, max_fuzzy_spelling_mistakes_num, pii_identification_method_drop_tabular, in_redact_comprehend_entities, comprehend_query_number, aws_access_key_textbox, aws_secret_key_textbox, actual_time_taken_number, do_initial_clean, chosen_language_drop], outputs=[text_output_summary, text_output_file, text_output_file_list_state, text_tabular_files_done, log_files_output, log_files_output_list_state, actual_time_taken_number], api_name="redact_data")
+    success(fn=anonymise_files_with_open_text, inputs=[in_data_files, in_text, anon_strategy, in_colnames,  in_redact_entities, in_allow_list_state, text_tabular_files_done, text_output_summary, text_output_file_list_state, log_files_output_list_state, in_excel_sheets, first_loop_state, output_folder_textbox, in_deny_list_state, max_fuzzy_spelling_mistakes_num, pii_identification_method_drop_tabular, in_redact_comprehend_entities, comprehend_query_number, aws_access_key_textbox, aws_secret_key_textbox, actual_time_taken_number, do_initial_clean, chosen_language_drop], outputs=[text_output_summary, text_output_file, text_output_file_list_state, text_tabular_files_done, log_files_output, log_files_output_list_state, actual_time_taken_number, comprehend_query_number], api_name="redact_data")
 
     # If the output file count text box changes, keep going with redacting each data file until done
-    text_tabular_files_done.change(fn=anonymise_files_with_open_text, inputs=[in_data_files, in_text, anon_strat, in_colnames,  in_redact_entities, in_allow_list_state, text_tabular_files_done, text_output_summary, text_output_file_list_state, log_files_output_list_state, in_excel_sheets, second_loop_state, output_folder_textbox, in_deny_list_state, max_fuzzy_spelling_mistakes_num, pii_identification_method_drop_tabular, in_redact_comprehend_entities, comprehend_query_number, aws_access_key_textbox, aws_secret_key_textbox, actual_time_taken_number, do_initial_clean, chosen_language_drop], outputs=[text_output_summary, text_output_file, text_output_file_list_state, text_tabular_files_done, log_files_output, log_files_output_list_state, actual_time_taken_number]).\
+    text_tabular_files_done.change(fn=anonymise_files_with_open_text, inputs=[in_data_files, in_text, anon_strategy, in_colnames,  in_redact_entities, in_allow_list_state, text_tabular_files_done, text_output_summary, text_output_file_list_state, log_files_output_list_state, in_excel_sheets, second_loop_state, output_folder_textbox, in_deny_list_state, max_fuzzy_spelling_mistakes_num, pii_identification_method_drop_tabular, in_redact_comprehend_entities, comprehend_query_number, aws_access_key_textbox, aws_secret_key_textbox, actual_time_taken_number, do_initial_clean, chosen_language_drop], outputs=[text_output_summary, text_output_file, text_output_file_list_state, text_tabular_files_done, log_files_output, log_files_output_list_state, actual_time_taken_number, comprehend_query_number]).\
     success(fn = reveal_feedback_buttons, outputs=[data_feedback_radio, data_further_details_text, data_submit_feedback_btn, data_feedback_title])
 
     ###
@@ -1039,12 +1099,15 @@ with app:
             min_word_count_input,
             min_consecutive_pages_input,
             greedy_match_input,
-            combine_page_text_for_duplicates_bool
+            combine_page_text_for_duplicates_bool,
+            output_folder_textbox
         ],
         outputs=[
             results_df_preview,
             duplicate_files_out, 
-            full_duplicate_data_by_file
+            full_duplicate_data_by_file,
+            actual_time_taken_number,
+            task_textbox
         ]
     )
 
@@ -1076,15 +1139,13 @@ with app:
     
     # Event handlers
     in_tabular_duplicate_files.upload(
-        fn=update_tabular_column_choices,
-        inputs=[in_tabular_duplicate_files],
-        outputs=[tabular_text_columns]
-    )
+        fn=put_columns_in_df, inputs=[in_tabular_duplicate_files], outputs=[tabular_text_columns, in_excel_tabular_sheets])
+    
     
     find_tabular_duplicates_btn.click(
         fn=run_tabular_duplicate_detection,
-        inputs=[in_tabular_duplicate_files, tabular_duplicate_threshold, tabular_min_word_count, tabular_text_columns, output_folder_textbox, do_initial_clean_dup],
-        outputs=[tabular_results_df, tabular_cleaned_file, tabular_file_to_clean], api_name="tabular_clean_duplicates", show_progress_on=[tabular_results_df]
+        inputs=[in_tabular_duplicate_files, tabular_duplicate_threshold, tabular_min_word_count, tabular_text_columns, output_folder_textbox, do_initial_clean_dup, in_excel_tabular_sheets, remove_duplicate_rows],
+        outputs=[tabular_results_df, tabular_cleaned_file, tabular_file_to_clean, actual_time_taken_number, task_textbox], api_name="tabular_clean_duplicates", show_progress_on=[tabular_results_df]
     )
     
     tabular_results_df.select(
@@ -1095,7 +1156,7 @@ with app:
     
     clean_duplicates_btn.click(
         fn=clean_tabular_duplicates,
-        inputs=[tabular_file_to_clean, tabular_results_df, output_folder_textbox],
+        inputs=[tabular_file_to_clean, tabular_results_df, output_folder_textbox, in_excel_tabular_sheets],
         outputs=[tabular_cleaned_file]
     )
 
@@ -1182,15 +1243,15 @@ with app:
     pdf_callback = CSVLogger_custom(dataset_file_name=FEEDBACK_LOG_FILE_NAME)
     data_callback = CSVLogger_custom(dataset_file_name=FEEDBACK_LOG_FILE_NAME)
 
-    if DISPLAY_FILE_NAMES_IN_LOGS == 'True':
+    if DISPLAY_FILE_NAMES_IN_LOGS == True:
         # User submitted feedback for pdf redactions
         pdf_callback.setup([pdf_feedback_radio, pdf_further_details_text, doc_file_name_no_extension_textbox], FEEDBACK_LOGS_FOLDER)
         pdf_submit_feedback_btn.click(lambda *args: pdf_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=FEEDBACK_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_FEEDBACK_LOG_HEADERS, replacement_headers=CSV_FEEDBACK_LOG_HEADERS), [pdf_feedback_radio, pdf_further_details_text, doc_file_name_no_extension_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[pdf_further_details_text])
 
         # User submitted feedback for data redactions        
-        data_callback.setup([data_feedback_radio, data_further_details_text, data_full_file_name_textbox], FEEDBACK_LOGS_FOLDER)
-        data_submit_feedback_btn.click(lambda *args: data_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=FEEDBACK_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_FEEDBACK_LOG_HEADERS, replacement_headers=CSV_FEEDBACK_LOG_HEADERS), [data_feedback_radio, data_further_details_text, data_full_file_name_textbox], None, preprocess=False).\
+        data_callback.setup([data_feedback_radio, data_further_details_text, data_file_name_with_extension_textbox], FEEDBACK_LOGS_FOLDER)
+        data_submit_feedback_btn.click(lambda *args: data_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=FEEDBACK_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_FEEDBACK_LOG_HEADERS, replacement_headers=CSV_FEEDBACK_LOG_HEADERS), [data_feedback_radio, data_further_details_text, data_file_name_with_extension_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[data_further_details_text])
     else:
         # User submitted feedback for pdf redactions
@@ -1199,7 +1260,7 @@ with app:
         success(fn = upload_log_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[pdf_further_details_text])
 
         # User submitted feedback for data redactions
-        data_callback.setup([data_feedback_radio, data_further_details_text, data_full_file_name_textbox], FEEDBACK_LOGS_FOLDER)
+        data_callback.setup([data_feedback_radio, data_further_details_text, data_file_name_with_extension_textbox], FEEDBACK_LOGS_FOLDER)
         data_submit_feedback_btn.click(lambda *args: data_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=FEEDBACK_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_FEEDBACK_LOG_HEADERS, replacement_headers=CSV_FEEDBACK_LOG_HEADERS), [data_feedback_radio, data_further_details_text, placeholder_data_file_name_no_extension_textbox_for_logs], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[data_further_details_text])
 
@@ -1207,27 +1268,41 @@ with app:
     # Log processing usage - time taken for redaction queries, and also logs for queries to Textract/Comprehend
     usage_callback = CSVLogger_custom(dataset_file_name=USAGE_LOG_FILE_NAME)
 
-    if DISPLAY_FILE_NAMES_IN_LOGS == 'True':
-        usage_callback.setup([session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], USAGE_LOGS_FOLDER)
+    if DISPLAY_FILE_NAMES_IN_LOGS == True:
+        usage_callback.setup([session_hash_textbox, doc_file_name_no_extension_textbox, data_file_name_with_extension_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], USAGE_LOGS_FOLDER)
+
+        latest_file_completed_num.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, doc_file_name_no_extension_textbox, data_file_name_with_extension_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False, api_name="usage_logs").\
+        success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
-        latest_file_completed_num.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], None, preprocess=False, api_name="usage_logs").\
+        text_tabular_files_done.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, doc_file_name_no_extension_textbox, data_file_name_with_extension_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
-        text_tabular_files_done.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], None, preprocess=False).\
+        successful_textract_api_call_number.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, doc_file_name_no_extension_textbox, data_file_name_with_extension_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
-        successful_textract_api_call_number.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], None, preprocess=False).\
+        # Deduplication usage logging
+        duplicate_files_out.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, actual_time_taken_number, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
+        success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
+
+        tabular_results_df.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
     else:
-        usage_callback.setup([session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], USAGE_LOGS_FOLDER)
+        usage_callback.setup([session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], USAGE_LOGS_FOLDER)
+
+        latest_file_completed_num.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, placeholder_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
+        success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
-        latest_file_completed_num.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, placeholder_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], None, preprocess=False).\
+        text_tabular_files_done.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, placeholder_data_file_name_no_extension_textbox_for_logs,  actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
-        text_tabular_files_done.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, placeholder_data_file_name_no_extension_textbox_for_logs,  actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], None, preprocess=False).\
+        successful_textract_api_call_number.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, placeholder_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
-        successful_textract_api_call_number.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, placeholder_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call], None, preprocess=False).\
+        # Deduplication usage logging (when file names not displayed)
+        duplicate_files_out.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, placeholder_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, total_pdf_page_count, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
+        success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
+
+        tabular_results_df.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB, dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, placeholder_doc_file_name_no_extension_textbox_for_logs, blank_data_file_name_no_extension_textbox_for_logs, total_pdf_page_count, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop_tabular, comprehend_query_number, cost_code_choice_drop, handwrite_signature_checkbox, host_name_textbox, text_extract_method_radio, is_a_textract_api_call, task_textbox], None, preprocess=False).\
         success(fn = upload_log_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
 if __name__ == "__main__":
@@ -1252,46 +1327,62 @@ if __name__ == "__main__":
             'task': DIRECT_MODE_TASK,
             'input_file': DIRECT_MODE_INPUT_FILE,
             'output_dir': DIRECT_MODE_OUTPUT_DIR,
+            'input_dir': INPUT_FOLDER,
             'language': DEFAULT_LANGUAGE,
             'allow_list': ALLOW_LIST_PATH,
             'pii_detector': LOCAL_PII_OPTION,
+            'username': DIRECT_MODE_DEFAULT_USER,
+            'save_to_user_folders': SESSION_OUTPUT_FOLDER,
             'aws_access_key': AWS_ACCESS_KEY,
             'aws_secret_key': AWS_SECRET_KEY,
             'aws_region': AWS_REGION,
             's3_bucket': DOCUMENT_REDACTION_BUCKET,
             'do_initial_clean': DO_INITIAL_TABULAR_DATA_CLEAN,
             'save_logs_to_csv': SAVE_LOGS_TO_CSV,
+            'save_logs_to_dynamodb': SAVE_LOGS_TO_DYNAMODB,
             'display_file_names_in_logs': DISPLAY_FILE_NAMES_IN_LOGS,
+            'upload_logs_to_s3': RUN_AWS_FUNCTIONS == "1",
+            's3_logs_prefix': S3_USAGE_LOGS_FOLDER,
             'ocr_method': TESSERACT_TEXT_EXTRACT_OPTION,
             'page_min': DEFAULT_PAGE_MIN,
             'page_max': DEFAULT_PAGE_MAX,
-            'prepare_for_review': False,
-            'prepare_images': True,
-            'no_images': False,
             'images_dpi': IMAGES_DPI,
-            'max_image_pixels': None,
-            'load_truncated_images': True,
             'chosen_local_ocr_model': CHOSEN_LOCAL_OCR_MODEL,
             'preprocess_local_ocr_images': PREPROCESS_LOCAL_OCR_IMAGES,
             'compress_redacted_pdf': COMPRESS_REDACTED_PDF,
             'return_pdf_end_of_redaction': RETURN_PDF_END_OF_REDACTION,
-            'in_allow_list': OUTPUT_ALLOW_LIST_PATH,
-            'in_deny_list': OUTPUT_DENY_LIST_PATH,
-            'redact_whole_page_list': OUTPUT_WHOLE_PAGE_REDACTION_LIST_PATH,
-            'handwrite_signature_checkbox': DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX,
-            'anon_strat': DEFAULT_TABULAR_ANONYMISATION_STRATEGY,
-            'columns': DEFAULT_TEXT_COLUMNS,
+            'allow_list_file': ALLOW_LIST_PATH,
+            'deny_list_file': DENY_LIST_PATH,
+            'redact_whole_page_file': WHOLE_PAGE_REDACTION_LIST_PATH,
+            'handwrite_signature_extraction': DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX,
+            'anon_strategy': DEFAULT_TABULAR_ANONYMISATION_STRATEGY,
             'excel_sheets': DEFAULT_EXCEL_SHEETS,
-            'deny_list': OUTPUT_DENY_LIST_PATH,
             'fuzzy_mistakes': DEFAULT_FUZZY_SPELLING_MISTAKES_NUM,
+            'match_fuzzy_whole_phrase_bool': 'True',  # Default value
             'duplicate_type': DIRECT_MODE_DUPLICATE_TYPE,
             'similarity_threshold': DEFAULT_DUPLICATE_DETECTION_THRESHOLD,
             'min_word_count': DEFAULT_MIN_WORD_COUNT,
             'min_consecutive_pages': DEFAULT_MIN_CONSECUTIVE_PAGES,
             'greedy_match': USE_GREEDY_DUPLICATE_DETECTION,
             'combine_pages': DEFAULT_COMBINE_PAGES,
-            'search_query': DEFAULT_SEARCH_QUERY if DEFAULT_SEARCH_QUERY else None,
-            'text_columns': DEFAULT_TEXT_COLUMNS.split(',') if DEFAULT_TEXT_COLUMNS else []
+            'search_query': DEFAULT_SEARCH_QUERY,
+            'text_columns': DEFAULT_TEXT_COLUMNS,
+            'remove_duplicate_rows': REMOVE_DUPLICATE_ROWS,
+            # Textract specific arguments (with defaults)
+            'textract_action': '',
+            'job_id': '',
+            'extract_signatures': False,
+            'textract_bucket': TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET,
+            'textract_input_prefix': TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER,
+            'textract_output_prefix': TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER,
+            's3_textract_document_logs_subfolder': TEXTRACT_JOBS_S3_LOC,
+            'local_textract_document_logs_subfolder': TEXTRACT_JOBS_LOCAL_LOC,
+            'poll_interval': 30,
+            'max_poll_attempts': 120,
+            # General arguments that might be missing
+            'local_redact_entities': CHOSEN_REDACT_ENTITIES,
+            'aws_redact_entities': CHOSEN_COMPREHEND_ENTITIES,
+            'cost_code': DEFAULT_COST_CODE
         }
         
         print(f"Running in direct mode with task: {DIRECT_MODE_TASK}")
@@ -1306,6 +1397,6 @@ if __name__ == "__main__":
                 print(f"Search query: {DEFAULT_SEARCH_QUERY}")
             if DEFAULT_TEXT_COLUMNS:
                 print(f"Text columns: {DEFAULT_TEXT_COLUMNS}")
-        
+            print(f"Remove duplicate rows: {REMOVE_DUPLICATE_ROWS}")
         # Run the CLI main function with direct mode arguments
         main(direct_mode_args=direct_mode_args)

cli_redact.py

@@ -1,15 +1,63 @@
 import argparse
 import os
 import pandas as pd
-from tools.config import get_or_create_env_var, LOCAL_PII_OPTION, AWS_PII_OPTION, SELECTABLE_TEXT_EXTRACT_OPTION, TESSERACT_TEXT_EXTRACT_OPTION, TEXTRACT_TEXT_EXTRACT_OPTION, INPUT_FOLDER, OUTPUT_FOLDER, DEFAULT_LANGUAGE, CHOSEN_COMPREHEND_ENTITIES, FULL_COMPREHEND_ENTITY_LIST, CHOSEN_REDACT_ENTITIES, FULL_ENTITY_LIST
+import time
+import uuid
+from tools.config import LOCAL_PII_OPTION, AWS_PII_OPTION, OUTPUT_FOLDER, DEFAULT_LANGUAGE, CHOSEN_COMPREHEND_ENTITIES, FULL_COMPREHEND_ENTITY_LIST, CHOSEN_REDACT_ENTITIES, FULL_ENTITY_LIST, CUSTOM_ENTITIES, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, DOCUMENT_REDACTION_BUCKET, DEFAULT_COST_CODE, SAVE_LOGS_TO_CSV, SAVE_LOGS_TO_DYNAMODB, DISPLAY_FILE_NAMES_IN_LOGS, DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX, DO_INITIAL_TABULAR_DATA_CLEAN, ALLOW_LIST_PATH, DENY_LIST_PATH, WHOLE_PAGE_REDACTION_LIST_PATH, PREPROCESS_LOCAL_OCR_IMAGES, IMAGES_DPI, RETURN_PDF_END_OF_REDACTION, COMPRESS_REDACTED_PDF, CHOSEN_LOCAL_OCR_MODEL, DEFAULT_TABULAR_ANONYMISATION_STRATEGY, DEFAULT_FUZZY_SPELLING_MISTAKES_NUM, DEFAULT_DUPLICATE_DETECTION_THRESHOLD, DEFAULT_MIN_WORD_COUNT, DEFAULT_MIN_CONSECUTIVE_PAGES, USE_GREEDY_DUPLICATE_DETECTION, DEFAULT_COMBINE_PAGES, REMOVE_DUPLICATE_ROWS, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER, INPUT_FOLDER, TEXTRACT_JOBS_S3_LOC, TEXTRACT_JOBS_LOCAL_LOC, SESSION_OUTPUT_FOLDER, DIRECT_MODE_DEFAULT_USER, RUN_AWS_FUNCTIONS, S3_USAGE_LOGS_FOLDER
+
 from tools.helper_functions import ensure_output_folder_exists
-from tools.file_conversion import prepare_image_or_pdf
-from tools.file_redaction import choose_and_run_redactor
-from tools.data_anonymise import anonymise_files_with_open_text
-from tools.helper_functions import _get_env_list
-from tools.load_spacy_model_custom_recognisers import custom_entities
-from tools.find_duplicate_pages import run_duplicate_analysis, run_full_search_and_analysis
-from tools.find_duplicate_tabular import run_tabular_duplicate_analysis
+
+
+def _generate_session_hash() -> str:
+    """Generate a unique session hash for logging purposes."""
+    return str(uuid.uuid4())[:8]
+
+def get_username_and_folders(username:str = "",
+                                output_folder_textbox:str=OUTPUT_FOLDER,
+                                input_folder_textbox:str=INPUT_FOLDER,
+                                session_output_folder:str=SESSION_OUTPUT_FOLDER,
+                                textract_document_upload_input_folder:str=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER,
+                                textract_document_upload_output_folder:str=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER,
+                                s3_textract_document_logs_subfolder:str=TEXTRACT_JOBS_S3_LOC,
+                                local_textract_document_logs_subfolder:str=TEXTRACT_JOBS_LOCAL_LOC):
+
+
+    # Generate session hash for logging. Either from input user name or generated
+    if username:
+        out_session_hash = username
+    else:
+        out_session_hash = _generate_session_hash() 
+
+
+    if session_output_folder == 'True' or session_output_folder == True:
+        output_folder = output_folder_textbox + out_session_hash + "/"
+        input_folder = input_folder_textbox + out_session_hash + "/"
+
+        textract_document_upload_input_folder = textract_document_upload_input_folder + "/" + out_session_hash
+        textract_document_upload_output_folder = textract_document_upload_output_folder + "/" + out_session_hash
+
+        s3_textract_document_logs_subfolder = s3_textract_document_logs_subfolder + "/" + out_session_hash
+        local_textract_document_logs_subfolder = local_textract_document_logs_subfolder + "/" + out_session_hash + "/"
+
+    else:
+        output_folder = output_folder_textbox
+        input_folder = input_folder_textbox
+
+    if not os.path.exists(output_folder): os.mkdir(output_folder)
+    if not os.path.exists(input_folder): os.mkdir(input_folder)
+
+    return out_session_hash, output_folder, out_session_hash, input_folder, textract_document_upload_input_folder, textract_document_upload_output_folder, s3_textract_document_logs_subfolder, local_textract_document_logs_subfolder
+
+
+def _get_env_list(env_var_name: str) -> list[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('\"', '').replace("\'","")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(',') if s.strip()]
+
+
 
 # --- Constants and Configuration ---
 
@@ -17,18 +65,21 @@ if CHOSEN_COMPREHEND_ENTITIES: CHOSEN_COMPREHEND_ENTITIES = _get_env_list(CHOSEN
 if FULL_COMPREHEND_ENTITY_LIST: FULL_COMPREHEND_ENTITY_LIST = _get_env_list(FULL_COMPREHEND_ENTITY_LIST)
 if CHOSEN_REDACT_ENTITIES: CHOSEN_REDACT_ENTITIES = _get_env_list(CHOSEN_REDACT_ENTITIES)
 if FULL_ENTITY_LIST: FULL_ENTITY_LIST = _get_env_list(FULL_ENTITY_LIST)
+if CUSTOM_ENTITIES: CUSTOM_ENTITIES = _get_env_list(CUSTOM_ENTITIES)
+if DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX: DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX = _get_env_list(DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX)
 
 # Add custom spacy recognisers to the Comprehend list, so that local Spacy model can be used to pick up e.g. titles, streetnames, UK postcodes that are sometimes missed by comprehend
-CHOSEN_COMPREHEND_ENTITIES.extend(custom_entities)
-FULL_COMPREHEND_ENTITY_LIST.extend(custom_entities)
+CHOSEN_COMPREHEND_ENTITIES.extend(CUSTOM_ENTITIES)
+FULL_COMPREHEND_ENTITY_LIST.extend(CUSTOM_ENTITIES)
 
 chosen_redact_entities = CHOSEN_REDACT_ENTITIES
 full_entity_list = FULL_ENTITY_LIST
 chosen_comprehend_entities = CHOSEN_COMPREHEND_ENTITIES
 full_comprehend_entity_list = FULL_COMPREHEND_ENTITY_LIST
+default_handwrite_signature_checkbox = DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX
 
 # --- Main CLI Function ---
-def main(direct_mode_args=None):
+def main(direct_mode_args={}):
     """
     A unified command-line interface to prepare, redact, and anonymise various document types.
     
@@ -40,99 +91,159 @@ def main(direct_mode_args=None):
         description='A versatile CLI for redacting PII from PDF/image files and anonymising Word/tabular data.',
         formatter_class=argparse.RawTextHelpFormatter,
         epilog='''
-Examples:
-  # Redact a PDF with default settings:
-  python cli_redact.py --input_file document.pdf
+Examples (look in the output/ folder to see output files):
+
+# Redaction
+
+## Redact a PDF with default settings (local OCR):
+python cli_redact.py --input_file example_data/example_of_emails_sent_to_a_professor_before_applying.pdf
+
+## Extract text from a PDF only (i.e. no redaction), using local OCR:
+python cli_redact.py --input_file example_data/Partnership-Agreement-Toolkit_0_0.pdf --redact_whole_page_file example_data/partnership_toolkit_redact_some_pages.csv --pii_detector None
+
+## Extract text from a PDF only (i.e. no redaction), using local OCR, with a whole page redaction list:
+python cli_redact.py --input_file example_data/Partnership-Agreement-Toolkit_0_0.pdf --redact_whole_page_file example_data/partnership_toolkit_redact_some_pages.csv --pii_detector Local --local_redact_entities CUSTOM
+
+## Redact a PDF with allow list (local OCR) and custom list of redaction entities:
+python cli_redact.py --input_file example_data/graduate-job-example-cover-letter.pdf --allow_list_file example_data/test_allow_list_graduate.csv --local_redact_entities TITLES PERSON DATE_TIME
 
-  # Redact specific pages with custom OCR:
-  python cli_redact.py --input_file document.pdf --page_min 1 --page_max 10 --ocr_method "AWS Textract service - all PDF types"
+## Redact a PDF with limited pages and text extraction method (local text) with custom fuzzy matching:
+python cli_redact.py --input_file example_data/Partnership-Agreement-Toolkit_0_0.pdf --deny_list_file example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv --local_redact_entities CUSTOM_FUZZY --page_min 1 --page_max 3 --ocr_method "Local text" --fuzzy_mistakes 3
 
-  # Anonymize Excel file with specific columns:
-  python cli_redact.py --input_file data.xlsx --columns "Name" "Email" --anon_strat "replace with 'REDACTED'"
+## Redaction with custom deny list, allow list, and whole page redaction list:
+python cli_redact.py --input_file example_data/Partnership-Agreement-Toolkit_0_0.pdf --deny_list_file example_data/partnership_toolkit_redact_custom_deny_list.csv --redact_whole_page_file example_data/partnership_toolkit_redact_some_pages.csv --allow_list_file example_data/test_allow_list_partnership.csv
 
-  # Use AWS services with custom settings:
-  python cli_redact.py --input_file document.pdf --pii_detector "AWS Comprehend" --aws_access_key YOUR_KEY --aws_secret_key YOUR_SECRET
+## Redact an image:
+python cli_redact.py --input_file example_data/example_complaint_letter.jpg
 
-  # Advanced redaction with custom word list:
-  python cli_redact.py --input_file document.pdf --in_deny_list "CompanyName" "ProjectCode" --deny_list custom_terms.csv
+## Anonymise csv file with specific columns:
+python cli_redact.py --input_file example_data/combined_case_notes.csv --text_columns "Case Note" "Client" --anon_strategy replace_redacted
 
-  # Find duplicate pages in OCR files:
-  python cli_redact.py --task deduplicate --input_file ocr_output.csv --duplicate_type pages --similarity_threshold 0.95
+## Anonymise csv file with a different strategy (remove text completely):
+python cli_redact.py --input_file example_data/combined_case_notes.csv --text_columns "Case Note" "Client" --anon_strategy redact
 
-  # Find duplicate content with search query:
-  python cli_redact.py --task deduplicate --input_file ocr_output.csv --duplicate_type pages --search_query "confidential information"
+## Anonymise a word document:
+python cli_redact.py --input_file "example_data/Bold minimalist professional cover letter.docx" --anon_strategy replace_redacted
 
-  # Find duplicate rows in tabular data:
-  python cli_redact.py --task deduplicate --input_file data.csv --duplicate_type tabular --text_columns "Name" "Description"
-        '''
+# Redaction with AWS services:
+
+## Use Textract and Comprehend::
+python cli_redact.py --input_file example_data/example_of_emails_sent_to_a_professor_before_applying.pdf --ocr_method "AWS Textract" --pii_detector "AWS Comprehend"
+
+## Redact specific pages with AWS OCR and signature extraction:
+python cli_redact.py --input_file example_data/Partnership-Agreement-Toolkit_0_0.pdf --page_min 6 --page_max 7 --ocr_method "AWS Textract" --handwrite_signature_extraction "Extract handwriting" "Extract signatures"
+
+# Duplicate page detection
+
+## Find duplicate pages in OCR files:
+python cli_redact.py --task deduplicate --input_file example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv --duplicate_type pages --similarity_threshold 0.95
+
+## Find duplicate in OCR files at the line level:
+python cli_redact.py --task deduplicate --input_file example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv --duplicate_type pages --similarity_threshold 0.95 --combine_pages False --min_word_count 3
+
+## Find duplicate rows in tabular data:
+python cli_redact.py --task deduplicate --input_file example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv --duplicate_type tabular --text_columns "text" --similarity_threshold 0.95
+
+# AWS Textract whole document analysis
+
+## Submit document to Textract for basic text analysis:
+python cli_redact.py --task textract --textract_action submit --input_file example_data/example_of_emails_sent_to_a_professor_before_applying.pdf
+
+## Submit document to Textract for analysis with signature extraction (Job ID will be printed to the console, you need this to retrieve the results):
+python cli_redact.py --task textract --textract_action submit --input_file example_data/Partnership-Agreement-Toolkit_0_0.pdf --extract_signatures 
+
+## Retrieve Textract results by job ID (returns a .json file output):
+python cli_redact.py --task textract --textract_action retrieve --job_id 12345678-1234-1234-1234-123456789012
+
+## List recent Textract jobs:
+python cli_redact.py --task textract --textract_action list
+
+'''
     )
 
     # --- Task Selection ---
     task_group = parser.add_argument_group('Task Selection')
     task_group.add_argument('--task', 
-    choices=['redact', 'deduplicate'], 
+    choices=['redact', 'deduplicate', 'textract'], 
     default='redact', 
-    help='Task to perform: redact (PII redaction/anonymization) or deduplicate (find duplicate content).')
+    help='Task to perform: redact (PII redaction/anonymisation), deduplicate (find duplicate content), or textract (AWS Textract batch operations).')
 
     # --- General Arguments (apply to all file types) ---
     general_group = parser.add_argument_group('General Options')
-    general_group.add_argument('--input_file', required=True, help='Path to the input file to process.')
+    general_group.add_argument('--input_file',  nargs='+', help='Path to the input file(s) to process. Separate multiple files with a space, and use quotes if there are spaces in the file name.')
     general_group.add_argument('--output_dir', default=OUTPUT_FOLDER, help='Directory for all output files.')
+    general_group.add_argument('--input_dir', default=INPUT_FOLDER, help='Directory for all input files.')
     general_group.add_argument('--language', default=DEFAULT_LANGUAGE, help='Language of the document content.')
-    general_group.add_argument('--allow_list', help='Path to a CSV file with words to exclude from redaction.')
-    general_group.add_argument('--pii_detector', 
-    choices=[LOCAL_PII_OPTION, AWS_PII_OPTION], 
-    default=LOCAL_PII_OPTION, 
-    help='Core PII detection method (Local or AWS).')
-    general_group.add_argument('--aws_access_key', default='', help='Your AWS Access Key ID.')
-    general_group.add_argument('--aws_secret_key', default='', help='Your AWS Secret Access Key.')
-    general_group.add_argument('--aws_region', default='', help='AWS region for cloud services.')
-    general_group.add_argument('--s3_bucket', default='', help='S3 bucket name for cloud operations.')
-    general_group.add_argument('--do_initial_clean', action='store_true', help='Perform initial text cleaning for tabular data.')
-    general_group.add_argument('--save_logs_to_csv', action='store_true', help='Save processing logs to CSV files.')
-    general_group.add_argument('--display_file_names_in_logs', action='store_true', help='Include file names in log outputs.')
+    general_group.add_argument('--allow_list', default=ALLOW_LIST_PATH, help='Path to a CSV file with words to exclude from redaction.')
+    general_group.add_argument('--pii_detector', choices=[LOCAL_PII_OPTION, AWS_PII_OPTION, "None"], default=LOCAL_PII_OPTION, 
+    help='Core PII detection method (Local or AWS Comprehend, or None).')
+    general_group.add_argument('--username', default=DIRECT_MODE_DEFAULT_USER, help='Username for the session.')
+    general_group.add_argument('--save_to_user_folders', default=SESSION_OUTPUT_FOLDER, help='Whether to save to user folders or not.')
+
+    general_group.add_argument('--local_redact_entities', nargs='+', choices=full_entity_list, default=chosen_redact_entities, 
+    help=f'Local redaction entities to use. Default: {chosen_redact_entities}. Full list: {full_entity_list}.')
+
+    general_group.add_argument('--aws_redact_entities', nargs='+', choices=full_comprehend_entity_list, default=chosen_comprehend_entities, 
+    help=f'AWS redaction entities to use. Default: {chosen_comprehend_entities}. Full list: {full_comprehend_entity_list}.')
+
+    general_group.add_argument('--aws_access_key', default=AWS_ACCESS_KEY, help='Your AWS Access Key ID.')
+    general_group.add_argument('--aws_secret_key', default=AWS_SECRET_KEY, help='Your AWS Secret Access Key.')
+    general_group.add_argument('--cost_code', default=DEFAULT_COST_CODE, help='Cost code for tracking usage.')
+    general_group.add_argument('--aws_region', default=AWS_REGION, help='AWS region for cloud services.')
+    general_group.add_argument('--s3_bucket', default=DOCUMENT_REDACTION_BUCKET, help='S3 bucket name for cloud operations.')
+    general_group.add_argument('--do_initial_clean', default=DO_INITIAL_TABULAR_DATA_CLEAN, help='Perform initial text cleaning for tabular data.')
+    general_group.add_argument('--save_logs_to_csv', default=SAVE_LOGS_TO_CSV, help='Save processing logs to CSV files.')
+    general_group.add_argument('--save_logs_to_dynamodb', default=SAVE_LOGS_TO_DYNAMODB, help='Save processing logs to DynamoDB.')
+    general_group.add_argument('--display_file_names_in_logs', default=DISPLAY_FILE_NAMES_IN_LOGS, help='Include file names in log outputs.')
+    general_group.add_argument('--upload_logs_to_s3', default=RUN_AWS_FUNCTIONS == "1", help='Upload log files to S3 after processing.')
+    general_group.add_argument('--s3_logs_prefix', default=S3_USAGE_LOGS_FOLDER, help='S3 prefix for usage log files.')
 
     # --- PDF/Image Redaction Arguments ---
     pdf_group = parser.add_argument_group('PDF/Image Redaction Options (.pdf, .png, .jpg)')
-    pdf_group.add_argument('--ocr_method', 
-    choices=[SELECTABLE_TEXT_EXTRACT_OPTION, TESSERACT_TEXT_EXTRACT_OPTION, TEXTRACT_TEXT_EXTRACT_OPTION], 
-    default=TESSERACT_TEXT_EXTRACT_OPTION, 
-    help='OCR method for text extraction from images.')
+    pdf_group.add_argument('--ocr_method', choices=["AWS Textract", "Local OCR", "Local text"], default="Local OCR", help='OCR method for text extraction from images.')
     pdf_group.add_argument('--page_min', type=int, default=0, help='First page to redact.')
-    pdf_group.add_argument('--page_max', type=int, default=999, help='Last page to redact.')
-    pdf_group.add_argument('--prepare_for_review', action='store_true', help='Prepare files for reviewing redactions.')
-    pdf_group.add_argument('--prepare_images', action='store_true', default=True, help='Enable image creation for PDF pages.')
-    pdf_group.add_argument('--no_images', action='store_false', dest='prepare_images', help='Disable image creation for PDF pages.')
-    pdf_group.add_argument('--images_dpi', type=float, default=300.0, help='DPI for image processing.')
-    pdf_group.add_argument('--max_image_pixels', type=int, help='Maximum image pixels for processing.')
-    pdf_group.add_argument('--load_truncated_images', action='store_true', help='Load truncated images during processing.')
-    pdf_group.add_argument('--chosen_local_ocr_model', choices=['tesseract', 'hybrid', 'paddle'], default='tesseract', help='Local OCR model to use.')
-    pdf_group.add_argument('--preprocess_local_ocr_images', action='store_true', help='Preprocess images before OCR.')
-    pdf_group.add_argument('--compress_redacted_pdf', action='store_true', help='Compress the final redacted PDF.')
-    pdf_group.add_argument('--return_pdf_end_of_redaction', action='store_true', default=True, help='Return PDF at end of redaction process.')
-    pdf_group.add_argument('--in_deny_list', nargs='+', default=list(), help='Custom words to recognize for redaction.')
-    pdf_group.add_argument('--redact_whole_page_list', nargs='+', default=list(), help='Pages to redact completely.')
-    pdf_group.add_argument('--handwrite_signature_checkbox', nargs='+', default=['Extract handwriting', 'Extract signatures'], help='Handwriting and signature extraction options.')
+    pdf_group.add_argument('--page_max', type=int, default=0, help='Last page to redact.')
+    pdf_group.add_argument('--images_dpi', type=float, default=float(IMAGES_DPI), help='DPI for image processing.')
+    pdf_group.add_argument('--chosen_local_ocr_model', choices=['tesseract', 'hybrid', 'paddle'], default=CHOSEN_LOCAL_OCR_MODEL, help='Local OCR model to use.')
+    pdf_group.add_argument('--preprocess_local_ocr_images', default=PREPROCESS_LOCAL_OCR_IMAGES, help='Preprocess images before OCR.')
+    pdf_group.add_argument('--compress_redacted_pdf', default=COMPRESS_REDACTED_PDF, help='Compress the final redacted PDF.')
+    pdf_group.add_argument('--return_pdf_end_of_redaction', default=RETURN_PDF_END_OF_REDACTION, help='Return PDF at end of redaction process.')
+    pdf_group.add_argument('--deny_list_file', default=DENY_LIST_PATH, help='Custom words file to recognize for redaction.')
+    pdf_group.add_argument('--allow_list_file', default=ALLOW_LIST_PATH, help='Custom words file to recognize for redaction.')
+    pdf_group.add_argument('--redact_whole_page_file', default=WHOLE_PAGE_REDACTION_LIST_PATH, help='File for pages to redact completely.')
+    pdf_group.add_argument('--handwrite_signature_extraction', nargs='+', default=default_handwrite_signature_checkbox, help='Handwriting and signature extraction options. Choose from "Extract handwriting", "Extract signatures".')
 
     # --- Word/Tabular Anonymisation Arguments ---
     tabular_group = parser.add_argument_group('Word/Tabular Anonymisation Options (.docx, .csv, .xlsx)')
-    tabular_group.add_argument('--anon_strat', choices=['redact', 'encrypt', 'hash', 'replace with \'REDACTED\'', 'replace with <ENTITY_NAME>', 'redact completely', 'mask', 'fake_first_name'], default='redact', help='The anonymisation strategy to apply.')
-    tabular_group.add_argument('--columns', nargs='+', default=list(), help='A list of column names to anonymise in tabular data.')
+    tabular_group.add_argument('--anon_strategy', choices=['redact', 'redact completely', 'replace_redacted', 'entity_type', 'encrypt', 'hash', 'replace with \'REDACTED\'', 'replace with <ENTITY_NAME>',  'mask', 'fake_first_name'], default=DEFAULT_TABULAR_ANONYMISATION_STRATEGY, help='The anonymisation strategy to apply.')
+    tabular_group.add_argument('--text_columns', nargs='+', default=list(), help='A list of column names to anonymise or deduplicate in tabular data.')
     tabular_group.add_argument('--excel_sheets', nargs='+', default=list(), help='Specific Excel sheet names to process.')
-    tabular_group.add_argument('--deny_list', help='Path to a CSV file with specific terms/phrases to redact.')
-    tabular_group.add_argument('--fuzzy_mistakes', type=int, default=1, help='Number of allowed spelling mistakes for fuzzy matching.')
-
+    tabular_group.add_argument('--fuzzy_mistakes', type=int, default=DEFAULT_FUZZY_SPELLING_MISTAKES_NUM, help='Number of allowed spelling mistakes for fuzzy matching.')
+    tabular_group.add_argument('--match_fuzzy_whole_phrase_bool', default=True, help='Match fuzzy whole phrase boolean.')
     # --- Duplicate Detection Arguments ---
     duplicate_group = parser.add_argument_group('Duplicate Detection Options')
     duplicate_group.add_argument('--duplicate_type', choices=['pages', 'tabular'], default='pages', help='Type of duplicate detection: pages (for OCR files) or tabular (for CSV/Excel files).')
-    duplicate_group.add_argument('--similarity_threshold', type=float, default=0.95, help='Similarity threshold (0-1) to consider content as duplicates.')
-    duplicate_group.add_argument('--min_word_count', type=int, default=3, help='Minimum word count for text to be considered in duplicate analysis.')
-    duplicate_group.add_argument('--min_consecutive_pages', type=int, default=1, help='Minimum number of consecutive pages to consider as a match.')
-    duplicate_group.add_argument('--greedy_match', action='store_true', default=True, help='Use greedy matching strategy for consecutive pages.')
-    duplicate_group.add_argument('--combine_pages', action='store_true', default=True, help='Combine text from the same page number within a file.')
-    duplicate_group.add_argument('--search_query', help='Search query text to find specific duplicate content (for page duplicates).')
-    duplicate_group.add_argument('--text_columns', nargs='+', default=list(), help='Specific text columns to analyze for duplicates (for tabular data).')
+    duplicate_group.add_argument('--similarity_threshold', type=float, default=DEFAULT_DUPLICATE_DETECTION_THRESHOLD, help='Similarity threshold (0-1) to consider content as duplicates.')
+    duplicate_group.add_argument('--min_word_count', type=int, default=DEFAULT_MIN_WORD_COUNT, help='Minimum word count for text to be considered in duplicate analysis.')
+    duplicate_group.add_argument('--min_consecutive_pages', type=int, default=DEFAULT_MIN_CONSECUTIVE_PAGES, help='Minimum number of consecutive pages to consider as a match.')
+    duplicate_group.add_argument('--greedy_match', default=USE_GREEDY_DUPLICATE_DETECTION, help='Use greedy matching strategy for consecutive pages.')
+    duplicate_group.add_argument('--combine_pages', default=DEFAULT_COMBINE_PAGES, help='Combine text from the same page number within a file. Alternative will enable line-level duplicate detection.')
+    duplicate_group.add_argument('--remove_duplicate_rows', default=REMOVE_DUPLICATE_ROWS, help='Remove duplicate rows from the output.')
 
+    # --- Textract Batch Operations Arguments ---
+    textract_group = parser.add_argument_group('Textract Batch Operations Options')
+    textract_group.add_argument('--textract_action', 
+    choices=['submit', 'retrieve', 'list'], 
+    help='Textract action to perform: submit (submit document for analysis), retrieve (get results by job ID), or list (show recent jobs).')
+    textract_group.add_argument('--job_id', help='Textract job ID for retrieve action.')
+    textract_group.add_argument('--extract_signatures', action='store_true', help='Extract signatures during Textract analysis (for submit action).')
+    textract_group.add_argument('--textract_bucket', default=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, help='S3 bucket name for Textract operations (overrides default).')
+    textract_group.add_argument('--textract_input_prefix', default=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER, help='S3 prefix for input files in Textract operations.')
+    textract_group.add_argument('--textract_output_prefix', default=TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER, help='S3 prefix for output files in Textract operations.')
+    textract_group.add_argument('--s3_textract_document_logs_subfolder', default=TEXTRACT_JOBS_S3_LOC, help='S3 prefix for logs in Textract operations.')
+    textract_group.add_argument('--local_textract_document_logs_subfolder', default=TEXTRACT_JOBS_LOCAL_LOC, help='Local prefix for logs in Textract operations.')
+    textract_group.add_argument('--poll_interval', type=int, default=30, help='Polling interval in seconds for Textract job status.')
+    textract_group.add_argument('--max_poll_attempts', type=int, default=120, help='Maximum number of polling attempts for Textract job completion.')
     # Parse arguments - either from command line or direct mode
     if direct_mode_args:
         # Use direct mode arguments
@@ -142,54 +253,161 @@ Examples:
         args = parser.parse_args()
 
     # --- Initial Setup ---
-    ensure_output_folder_exists(args.output_dir)
-    _, file_extension = os.path.splitext(args.input_file)
-    file_extension = file_extension.lower()
+    # Convert string boolean variables to boolean
+    if args.preprocess_local_ocr_images == "True": args.preprocess_local_ocr_images = True
+    else: args.preprocess_local_ocr_images = False
+    if args.greedy_match == "True": args.greedy_match = True
+    else: args.greedy_match = False
+    if args.combine_pages == "True": args.combine_pages = True
+    else: args.combine_pages = False
+    if args.remove_duplicate_rows == "True": args.remove_duplicate_rows = True
+    else: args.remove_duplicate_rows = False
+    if args.return_pdf_end_of_redaction == "True": args.return_pdf_end_of_redaction = True
+    else: args.return_pdf_end_of_redaction = False
+    if args.compress_redacted_pdf == "True": args.compress_redacted_pdf = True
+    else: args.compress_redacted_pdf = False
+    if args.do_initial_clean == "True": args.do_initial_clean = True
+    else: args.do_initial_clean = False
+    if args.save_logs_to_csv == "True": args.save_logs_to_csv = True
+    else: args.save_logs_to_csv = False
+    if args.save_logs_to_dynamodb == "True": args.save_logs_to_dynamodb = True
+    else: args.save_logs_to_dynamodb = False
+    if args.display_file_names_in_logs == "True": args.display_file_names_in_logs = True
+    else: args.display_file_names_in_logs = False
+    if args.match_fuzzy_whole_phrase_bool == "True": args.match_fuzzy_whole_phrase_bool = True
+    else: args.match_fuzzy_whole_phrase_bool = False
+    if args.save_to_user_folders == "True": args.save_to_user_folders = True
+    else: args.save_to_user_folders = False
+
+    if args.task in ['redact', 'deduplicate']:
+        if args.input_file:
+            if isinstance(args.input_file, str):
+                args.input_file = [args.input_file]
+
+            _, file_extension = os.path.splitext(args.input_file[0])
+            file_extension = file_extension.lower()
+        else:
+            raise ValueError("Error: --input_file is required for 'redact' task.")
     
-    # Load allow/deny lists
-    allow_list = pd.read_csv(args.allow_list) if args.allow_list else pd.DataFrame()
-    deny_list = pd.read_csv(args.deny_list).iloc[:, 0].tolist() if args.deny_list else []
+    # Initialise usage logger if logging is enabled
+    usage_logger = None
+    if args.save_logs_to_csv or args.save_logs_to_dynamodb:
+        from tools.cli_usage_logger import create_cli_usage_logger
+        try:
+            usage_logger = create_cli_usage_logger()
+        except Exception as e:
+            print(f"Warning: Could not initialise usage logger: {e}")
+
+    print(f"Argument args.save_to_user_folders: {args.save_to_user_folders} will be used to determine if outputs will be saved to user folders.")
+
+    # Get username and folders
+    session_hash, args.output_dir, _, args.input_dir, args.textract_input_prefix, args.textract_output_prefix, args.s3_textract_document_logs_subfolder, args.local_textract_document_logs_subfolder = get_username_and_folders(username=args.username, output_folder_textbox=args.output_dir, input_folder_textbox=args.input_dir, session_output_folder=args.save_to_user_folders, textract_document_upload_input_folder=args.textract_input_prefix, textract_document_upload_output_folder=args.textract_output_prefix, s3_textract_document_logs_subfolder=args.s3_textract_document_logs_subfolder, local_textract_document_logs_subfolder=args.local_textract_document_logs_subfolder)
+
+    print(f"Conducting analyses with user {args.username}. Outputs will be saved to {args.output_dir}.")
 
     # --- Route to the Correct Workflow Based on Task and File Type ---
 
-    # Task 1: Redaction/Anonymization
+    # Validate input_file requirement for tasks that need it
+    if args.task in ['redact', 'deduplicate'] and not args.input_file:
+        print(f"Error: --input_file is required for '{args.task}' task.")
+        return
+
+    if args.ocr_method in ["Local OCR", "AWS Textract"]:
+        args.prepare_images = True
+    else:
+        args.prepare_images = False
+
+    from tools.cli_usage_logger import create_cli_usage_logger, log_redaction_usage
+    # Task 1: Redaction/Anonymisation
     if args.task == 'redact':
+
         # Workflow 1: PDF/Image Redaction
         if file_extension in ['.pdf', '.png', '.jpg', '.jpeg']:
             print("--- Detected PDF/Image file. Starting Redaction Workflow... ---")
+            start_time = time.time()
             try:
+                from tools.file_conversion import prepare_image_or_pdf
+                from tools.file_redaction import choose_and_run_redactor
                 # Step 1: Prepare the document
                 print("\nStep 1: Preparing document...")
                 (
                     prep_summary, prepared_pdf_paths, image_file_paths, _, _, pdf_doc,
-                    image_annotations, _, original_cropboxes, page_sizes, textract_output_found, _, _, _, _
+                    image_annotations, _, original_cropboxes, page_sizes, _, _, _, _, _
                 ) = prepare_image_or_pdf(
-                    file_paths=[args.input_file], text_extract_method=args.ocr_method,
-                    all_line_level_ocr_results_df=pd.DataFrame(), all_page_line_level_ocr_results_with_words_df=pd.DataFrame(),
-                    first_loop_state=True, prepare_for_review=args.prepare_for_review,
-                    output_folder=args.output_dir, prepare_images=args.prepare_images
+                    file_paths=args.input_file, text_extract_method=args.ocr_method, all_line_level_ocr_results_df=pd.DataFrame(), all_page_line_level_ocr_results_with_words_df=pd.DataFrame(),
+                    first_loop_state=True, prepare_for_review=False,
+                    output_folder=args.output_dir, input_folder=args.input_dir, prepare_images=args.prepare_images
                 )
                 print(f"Preparation complete. {prep_summary}")
 
                 # Step 2: Redact the prepared document
                 print("\nStep 2: Running redaction...")
                 (
-                    output_summary, output_files, _, _, log_files, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _
+                    output_summary, output_files, _, _, log_files, _, _, _, _, _, _, _, _, _, comprehend_query_number, _, _, _, _, _, _, page_sizes, _, _, _, total_textract_query_number, _, _, _, _, _, _
                 ) = choose_and_run_redactor(
-                    file_paths=[args.input_file], prepared_pdf_file_paths=prepared_pdf_paths,
-                    pdf_image_file_paths=image_file_paths, chosen_redact_entities=chosen_redact_entities,
-                    chosen_redact_comprehend_entities=chosen_comprehend_entities, text_extraction_method=args.ocr_method,
-                    in_allow_list=allow_list, in_deny_list=args.in_deny_list,
-                    redact_whole_page_list=args.redact_whole_page_list, first_loop_state=True, 
-                    page_min=args.page_min, page_max=args.page_max, handwrite_signature_checkbox=args.handwrite_signature_checkbox,
+                    file_paths=args.input_file, prepared_pdf_file_paths=prepared_pdf_paths,
+                    pdf_image_file_paths=image_file_paths, chosen_redact_entities=args.local_redact_entities,
+                    chosen_redact_comprehend_entities=args.aws_redact_entities, text_extraction_method=args.ocr_method,
+                    in_allow_list=args.allow_list_file, in_deny_list=args.deny_list_file,
+                    redact_whole_page_list=args.redact_whole_page_file, first_loop_state=True, 
+                    page_min=args.page_min, page_max=args.page_max, handwrite_signature_checkbox=args.handwrite_signature_extraction, max_fuzzy_spelling_mistakes_num=args.fuzzy_mistakes, match_fuzzy_whole_phrase_bool=args.match_fuzzy_whole_phrase_bool,
                     pymupdf_doc=pdf_doc, annotations_all_pages=image_annotations, page_sizes=page_sizes,
                     document_cropboxes=original_cropboxes, pii_identification_method=args.pii_detector,
                     aws_access_key_textbox=args.aws_access_key, aws_secret_key_textbox=args.aws_secret_key,
-                    language=args.language, output_folder=args.output_dir
+                    language=args.language, output_folder=args.output_dir, input_folder=args.input_dir
                 )
+
+                # Calculate processing time
+                end_time = time.time()
+                processing_time = end_time - start_time
+                
+                # Log usage data if logger is available
+                if usage_logger:
+                    try:
+                        # Extract file name for logging
+                        print("Saving logs to CSV")
+                        doc_file_name = os.path.basename(args.input_file[0]) if args.display_file_names_in_logs else "document"
+                        data_file_name = ""  # Not applicable for PDF/image redaction
+                        
+                        # Determine if this was a Textract API call
+                        is_textract_call = args.ocr_method == "AWS Textract"
+                        
+                        # Count pages (approximate from page_sizes if available)
+                        total_pages = len(page_sizes) if page_sizes else 1
+                        
+                        # Count API calls (approximate - would need to be tracked in the redaction function)
+                        textract_queries = int(total_textract_query_number) if is_textract_call else 0
+                        comprehend_queries = int(comprehend_query_number) if args.pii_detector == "AWS Comprehend" else 0
+                        
+                        # Format handwriting/signature options
+                        handwriting_signature = ", ".join(args.handwrite_signature_extraction) if args.handwrite_signature_extraction else ""
+                        
+                        log_redaction_usage(
+                            logger=usage_logger,
+                            session_hash=session_hash,
+                            doc_file_name=doc_file_name,
+                            data_file_name=data_file_name,
+                            time_taken=processing_time,
+                            total_pages=total_pages,
+                            textract_queries=textract_queries,
+                            pii_method=args.pii_detector,
+                            comprehend_queries=comprehend_queries,
+                            cost_code=args.cost_code,
+                            handwriting_signature=handwriting_signature,
+                            text_extraction_method=args.ocr_method,
+                            is_textract_call=is_textract_call,
+                            task=args.task,
+                            save_to_dynamodb=args.save_logs_to_dynamodb,
+                            save_to_s3=args.upload_logs_to_s3,
+                            s3_bucket=args.s3_bucket,
+                            s3_key_prefix=args.s3_logs_prefix
+                        )
+                    except Exception as e:
+                        print(f"Warning: Could not log usage data: {e}")
                 
                 print("\n--- Redaction Process Complete ---")
                 print(f"Summary: {output_summary}")
+                print(f"Processing time: {processing_time:.2f} seconds")
                 print(f"\nOutput files saved to: {args.output_dir}")
                 print("Generated Files:", sorted(output_files))
                 if log_files: print("Log Files:", sorted(log_files))
@@ -200,30 +418,83 @@ Examples:
         # Workflow 2: Word/Tabular Data Anonymisation
         elif file_extension in ['.docx', '.xlsx', '.xls', '.csv', '.parquet']:
             print("--- Detected Word/Tabular file. Starting Anonymisation Workflow... ---")
+            start_time = time.time()
             try:
+                from tools.data_anonymise import anonymise_files_with_open_text
+                
                 # Run the anonymisation function directly
-                output_summary, output_files, _, _, log_files, _, _ = anonymise_files_with_open_text(
-                    file_paths=[args.input_file],
+
+                output_summary, output_files, _, _, log_files, _, processing_time, comprehend_query_number = anonymise_files_with_open_text(
+                    file_paths=args.input_file,
                     in_text="", # Not used for file-based operations
-                    anon_strat=args.anon_strat,
-                    chosen_cols=args.columns,
-                    chosen_redact_entities=chosen_redact_entities,
-                    in_allow_list=allow_list,
+                    anon_strategy=args.anon_strategy,
+                    chosen_cols=args.text_columns,
+                    chosen_redact_entities=args.local_redact_entities,
+                    in_allow_list=args.allow_list_file,
                     in_excel_sheets=args.excel_sheets,
                     first_loop_state=True,
                     output_folder=args.output_dir,
-                    in_deny_list=deny_list,
+                    in_deny_list=args.deny_list_file,
                     max_fuzzy_spelling_mistakes_num=args.fuzzy_mistakes,
                     pii_identification_method=args.pii_detector,
-                    chosen_redact_comprehend_entities=chosen_comprehend_entities,
+                    chosen_redact_comprehend_entities=args.aws_redact_entities,
                     aws_access_key_textbox=args.aws_access_key,
                     aws_secret_key_textbox=args.aws_secret_key,
                     language=args.language,
                     do_initial_clean=args.do_initial_clean
                 )
 
+                # Calculate processing time
+                end_time = time.time()
+                processing_time = end_time - start_time
+                
+                # Log usage data if logger is available
+                if usage_logger:
+                    try:
+                        print("Saving logs to CSV")
+                        # Extract file name for logging
+                        doc_file_name = ""  # Not applicable for tabular data
+                        data_file_name = os.path.basename(args.input_file[0]) if args.display_file_names_in_logs else "data_file"
+                        
+                        # Determine if this was a Textract API call (not applicable for tabular)
+                        is_textract_call = False
+                        
+                        # Count pages (not applicable for tabular data)
+                        total_pages = 0
+                        
+                        # Count API calls (approximate - would need to be tracked in the anonymisation function)
+                        textract_queries = 0  # Not applicable for tabular data
+                        comprehend_queries = comprehend_query_number if args.pii_detector == "AWS Comprehend" else 0
+                        
+                        # Format handwriting/signature options (not applicable for tabular)
+                        handwriting_signature = ""
+                        
+                        log_redaction_usage(
+                            logger=usage_logger,
+                            session_hash=session_hash,
+                            doc_file_name=doc_file_name,
+                            data_file_name=data_file_name,
+                            time_taken=processing_time,
+                            total_pages=total_pages,
+                            textract_queries=textract_queries,
+                            pii_method=args.pii_detector,
+                            comprehend_queries=comprehend_queries,
+                            cost_code=args.cost_code,
+                            handwriting_signature=handwriting_signature,
+                            text_extraction_method="tabular",  # Indicate this is tabular processing
+                            is_textract_call=is_textract_call,
+                            task=args.task,
+                            save_to_dynamodb=args.save_logs_to_dynamodb,
+                            save_to_s3=args.upload_logs_to_s3,
+                            s3_bucket=args.s3_bucket,
+                            s3_key_prefix=args.s3_logs_prefix
+                        )
+                    except Exception as e:
+                        print(f"Warning: Could not log usage data: {e}")
+
                 print("\n--- Anonymisation Process Complete ---")
                 print(f"Summary: {output_summary}")
+                print(f"Processing time: {processing_time:.2f} seconds")
                 print(f"\nOutput files saved to: {args.output_dir}")
                 print("Generated Files:", sorted(output_files))
                 if log_files: print("Log Files:", sorted(log_files))
@@ -240,29 +511,33 @@ Examples:
     elif args.task == 'deduplicate':
         print("--- Starting Duplicate Detection Workflow... ---")
         try:
+            from tools.find_duplicate_pages import run_duplicate_analysis
             if args.duplicate_type == 'pages':
                 # Page duplicate detection
                 if file_extension == '.csv':
                     print("--- Detected OCR CSV file. Starting Page Duplicate Detection... ---")
                     
-                    if args.search_query:
-                        # Use search-based duplicate detection
-                        print(f"Searching for duplicates of: '{args.search_query}'")
-                        # Note: This would require the OCR data to be loaded first
-                        # For now, we'll use the general duplicate analysis
-                        print("Note: Search-based duplicate detection requires OCR data preparation.")
-                        print("Using general duplicate analysis instead.")
-                    
+                    start_time = time.time()
+
+                    if args.combine_pages == True:
+                        print("Combining pages...")
+                    else:
+                        print("Using line-level duplicate detection...")
+
                     # Load the CSV file as a list for the duplicate analysis function
-                    results_df, output_paths, full_data_by_file = run_duplicate_analysis(
-                        files=[args.input_file],
+                    results_df, output_paths, full_data_by_file, processing_time, task_textbox = run_duplicate_analysis(
+                        files=args.input_file,
                         threshold=args.similarity_threshold,
                         min_words=args.min_word_count,
                         min_consecutive=args.min_consecutive_pages,
                         greedy_match=args.greedy_match,
-                        combine_pages=args.combine_pages
+                        combine_pages=args.combine_pages,
+                        output_folder=args.output_dir
                     )
                     
+                    end_time = time.time()
+                    processing_time = end_time - start_time
+
                     print("\n--- Page Duplicate Detection Complete ---")
                     print(f"Found {len(results_df)} duplicate matches")
                     print(f"\nOutput files saved to: {args.output_dir}")
@@ -271,19 +546,116 @@ Examples:
                 else:
                     print(f"Error: Page duplicate detection requires CSV files with OCR data.")
                     print("Please provide a CSV file containing OCR output data.")
+
+                    # Log usage data if logger is available
+                    if usage_logger:
+                        try:
+                            # Extract file name for logging
+                            print("Saving logs to CSV")
+                            doc_file_name = os.path.basename(args.input_file[0]) if args.display_file_names_in_logs else "document"
+                            data_file_name = ""  # Not applicable for PDF/image redaction
+                            
+                            # Determine if this was a Textract API call
+                            is_textract_call = False
+                            
+                            # Count pages (approximate from page_sizes if available)
+                            total_pages = len(page_sizes) if page_sizes else 1
+                            
+                            # Count API calls (approximate - would need to be tracked in the redaction function)
+                            textract_queries = 0
+                            comprehend_queries = 0
+                            
+                            # Format handwriting/signature options
+                            handwriting_signature = ""
+                            
+                            log_redaction_usage(
+                                logger=usage_logger,
+                                session_hash=session_hash,
+                                doc_file_name=doc_file_name,
+                                data_file_name=data_file_name,
+                                time_taken=processing_time,
+                                total_pages=total_pages,
+                                textract_queries=textract_queries,
+                                pii_method=args.pii_detector,
+                                comprehend_queries=comprehend_queries,
+                                cost_code=args.cost_code,
+                                handwriting_signature=handwriting_signature,
+                                text_extraction_method=args.ocr_method,
+                                is_textract_call=is_textract_call,
+                                task=args.task,
+                                save_to_dynamodb=args.save_logs_to_dynamodb,
+                                save_to_s3=args.upload_logs_to_s3,
+                                s3_bucket=args.s3_bucket,
+                                s3_key_prefix=args.s3_logs_prefix
+                            )
+                        except Exception as e:
+                            print(f"Warning: Could not log usage data: {e}")
                     
             elif args.duplicate_type == 'tabular':
                 # Tabular duplicate detection
+                from tools.find_duplicate_tabular import run_tabular_duplicate_detection
                 if file_extension in ['.csv', '.xlsx', '.xls', '.parquet']:
                     print("--- Detected tabular file. Starting Tabular Duplicate Detection... ---")
+
+                    start_time = time.time()
                     
-                    results_df, output_paths, full_data_by_file = run_tabular_duplicate_analysis(
-                        files=[args.input_file],
+                    results_df, output_paths, full_data_by_file, processing_time, task_textbox = run_tabular_duplicate_detection(
+                        files=args.input_file,
                         threshold=args.similarity_threshold,
                         min_words=args.min_word_count,
-                        text_columns=args.text_columns if args.text_columns else None,
-                        output_folder=args.output_dir
+                        text_columns=args.text_columns,
+                        output_folder=args.output_dir,
+                        do_initial_clean_dup=args.do_initial_clean,
+                        in_excel_tabular_sheets=args.excel_sheets,
+                        remove_duplicate_rows=args.remove_duplicate_rows
                     )
+
+                    end_time = time.time()
+                    processing_time = end_time - start_time
+
+                    # Log usage data if logger is available
+                    if usage_logger:
+                        try:
+                            # Extract file name for logging
+                            print("Saving logs to CSV")
+                            doc_file_name = ""
+                            data_file_name = os.path.basename(args.input_file[0]) if args.display_file_names_in_logs else "data_file"
+                            
+                            # Determine if this was a Textract API call
+                            is_textract_call = False
+                            
+                            # Count pages (approximate from page_sizes if available)
+                            total_pages = len(page_sizes) if page_sizes else 1
+                            
+                            # Count API calls (approximate - would need to be tracked in the redaction function)
+                            textract_queries = 0
+                            comprehend_queries = 0
+                            
+                            # Format handwriting/signature options
+                            handwriting_signature = ""
+                            
+                            log_redaction_usage(
+                                logger=usage_logger,
+                                session_hash=session_hash,
+                                doc_file_name=doc_file_name,
+                                data_file_name=data_file_name,
+                                time_taken=processing_time,
+                                total_pages=total_pages,
+                                textract_queries=textract_queries,
+                                pii_method=args.pii_detector,
+                                comprehend_queries=comprehend_queries,
+                                cost_code=args.cost_code,
+                                handwriting_signature=handwriting_signature,
+                                text_extraction_method=args.ocr_method,
+                                is_textract_call=is_textract_call,
+                                task=args.task,
+                                save_to_dynamodb=args.save_logs_to_dynamodb,
+                                save_to_s3=args.upload_logs_to_s3,
+                                s3_bucket=args.s3_bucket,
+                                s3_key_prefix=args.s3_logs_prefix
+                            )
+                        except Exception as e:
+                            print(f"Warning: Could not log usage data: {e}")
                     
                     print("\n--- Tabular Duplicate Detection Complete ---")
                     print(f"Found {len(results_df)} duplicate matches")
@@ -299,10 +671,175 @@ Examples:
                 
         except Exception as e:
             print(f"\nAn error occurred during the duplicate detection workflow: {e}")
+
+    # Task 3: Textract Batch Operations
+    elif args.task == 'textract':
+        print("--- Starting Textract Batch Operations Workflow... ---")
+        
+        if not args.textract_action:
+            print("Error: --textract_action is required for textract task.")
+            print("Valid options: 'submit', 'retrieve', or 'list'")
+            return
+            
+        try:
+            if args.textract_action == 'submit':
+                from tools.textract_batch_call import analyse_document_with_textract_api, load_in_textract_job_details
+                # Submit document to Textract for analysis
+                if not args.input_file:
+                    print("Error: --input_file is required for submit action.")
+                    return
+                    
+                print(f"--- Submitting document to Textract: {args.input_file} ---")
+
+                start_time = time.time()
+                
+                # Load existing job details
+                job_df = load_in_textract_job_details(load_s3_jobs_loc=args.s3_textract_document_logs_subfolder, load_local_jobs_loc=args.local_textract_document_logs_subfolder)
+                
+                # Determine signature extraction options
+                signature_options = ['Extract handwriting', 'Extract signatures'] if args.extract_signatures else ['Extract handwriting']
+                
+                # Use configured bucket or override
+                textract_bucket = args.textract_bucket if args.textract_bucket else ""
+                
+                # Submit the job
+                result_message, job_id, job_type, successful_job_number, is_textract_call, total_pages, task_textbox = analyse_document_with_textract_api(
+                    local_pdf_path=args.input_file,
+                    s3_input_prefix=args.textract_input_prefix,
+                    s3_output_prefix=args.textract_output_prefix,
+                    job_df=job_df,
+                    s3_bucket_name=textract_bucket,
+                    general_s3_bucket_name=args.s3_bucket,
+                    local_output_dir=args.output_dir,
+                    analyse_signatures=signature_options,
+                    aws_region=args.aws_region
+                )
+
+                end_time = time.time()
+                processing_time = end_time - start_time
+                
+                print(f"\n--- Textract Job Submitted Successfully ---")
+                print(f"Job ID: {job_id}")
+                print(f"Job Type: {job_type}")
+                print(f"Message: {result_message}")
+                print(f"Results will be available in: {args.output_dir}")
+
+                # Log usage data if logger is available
+                if usage_logger:
+                    try:
+                        # Extract file name for logging
+                        print("Saving logs to CSV")
+                        doc_file_name = os.path.basename(args.input_file[0]) if args.display_file_names_in_logs else "document"
+                        data_file_name = ""
+                        
+                        # Determine if this was a Textract API call
+                        is_textract_call = True
+                        args.ocr_method == "AWS Textract"
+                        
+                        # Count API calls (approximate - would need to be tracked in the redaction function)
+                        textract_queries = total_pages
+                        comprehend_queries = 0
+                        
+                        # Format handwriting/signature options
+                        handwriting_signature = ""
+                        
+                        log_redaction_usage(
+                            logger=usage_logger,
+                            session_hash=session_hash,
+                            doc_file_name=doc_file_name,
+                            data_file_name=data_file_name,
+                            time_taken=processing_time,
+                            total_pages=total_pages,
+                            textract_queries=textract_queries,
+                            pii_method=args.pii_detector,
+                            comprehend_queries=comprehend_queries,
+                            cost_code=args.cost_code,
+                            handwriting_signature=handwriting_signature,
+                            text_extraction_method=args.ocr_method,
+                            is_textract_call=is_textract_call,
+                            task=args.task,
+                            save_to_dynamodb=args.save_logs_to_dynamodb,
+                            save_to_s3=args.upload_logs_to_s3,
+                            s3_bucket=args.s3_bucket,
+                            s3_key_prefix=args.s3_logs_prefix
+                        )
+                    except Exception as e:
+                        print(f"Warning: Could not log usage data: {e}")
+                
+            elif args.textract_action == 'retrieve':
+                print(f"--- Retrieving Textract results for Job ID: {args.job_id} ---")
+
+                from tools.textract_batch_call import poll_whole_document_textract_analysis_progress_and_download, load_in_textract_job_details
+                # Retrieve results by job ID
+                if not args.job_id:
+                    print("Error: --job_id is required for retrieve action.")
+                    return
+                
+                # Load existing job details to get job type
+                print("Loading existing job details...")
+                job_df = load_in_textract_job_details(load_s3_jobs_loc=args.s3_textract_document_logs_subfolder, load_local_jobs_loc=args.local_textract_document_logs_subfolder)
+                
+                # Find job type from the dataframe
+                job_type = "document_text_detection"  # default
+                if not job_df.empty and "job_id" in job_df.columns:
+                    matching_jobs = job_df.loc[job_df["job_id"] == args.job_id]
+                    if not matching_jobs.empty and "job_type" in matching_jobs.columns:
+                        job_type = matching_jobs.iloc[0]["job_type"]
+                
+                # Use configured bucket or override
+                textract_bucket = args.textract_bucket if args.textract_bucket else ""
+                
+                # Poll for completion and download results
+                print("Polling for completion and downloading results...")
+                downloaded_file_path, job_status, updated_job_df, output_filename = poll_whole_document_textract_analysis_progress_and_download(
+                    job_id=args.job_id,
+                    job_type_dropdown=job_type,
+                    s3_output_prefix=args.textract_output_prefix,
+                    pdf_filename="",  # Will be determined from job details
+                    job_df=job_df,
+                    s3_bucket_name=textract_bucket,
+                    load_s3_jobs_loc=args.s3_textract_document_logs_subfolder,
+                    load_local_jobs_loc=args.local_textract_document_logs_subfolder,
+                    local_output_dir=args.output_dir,
+                    poll_interval_seconds=args.poll_interval,
+                    max_polling_attempts=args.max_poll_attempts
+                )
+                
+                print(f"\n--- Textract Results Retrieved Successfully ---")
+                print(f"Job Status: {job_status}")
+                print(f"Downloaded File: {downloaded_file_path}")
+                #print(f"Output Filename: {output_filename}")
+                
+            elif args.textract_action == 'list':
+                from tools.textract_batch_call import load_in_textract_job_details
+                # List recent Textract jobs
+                print("--- Listing Recent Textract Jobs ---")
+                
+                job_df = load_in_textract_job_details(load_s3_jobs_loc=args.s3_textract_document_logs_subfolder, load_local_jobs_loc=args.local_textract_document_logs_subfolder)
+                
+                if job_df.empty:
+                    print("No recent Textract jobs found.")
+                else:
+                    print(f"\nFound {len(job_df)} recent Textract jobs:")
+                    print("-" * 80)
+                    for _, job in job_df.iterrows():
+                        print(f"Job ID: {job.get('job_id', 'N/A')}")
+                        print(f"File: {job.get('file_name', 'N/A')}")
+                        print(f"Type: {job.get('job_type', 'N/A')}")
+                        print(f"Signatures: {job.get('signature_extraction', 'N/A')}")
+                        print(f"Date: {job.get('job_date_time', 'N/A')}")
+                        print("-" * 80)
+                
+            else:
+                print(f"Error: Invalid textract_action '{args.textract_action}'.")
+                print("Valid options: 'submit', 'retrieve', or 'list'")
+                
+        except Exception as e:
+            print(f"\nAn error occurred during the Textract workflow: {e}")
     
     else:
         print(f"Error: Invalid task '{args.task}'.")
-        print("Valid options: 'redact' or 'deduplicate'")
+        print("Valid options: 'redact', 'deduplicate', or 'textract'")
 
 if __name__ == "__main__":
     main()

example_data/Bold minimalist professional cover letter.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8551ac157f350b2093e5d8c89f68474f613350074201cff6d52d5ed5ec28ff
+size 23992

example_data/Partnership-Agreement-Toolkit_0_0.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db46a784d7aaafb8d02acf8686523dd376400117d07926a5dcb51ceb69e3236
+size 426602

example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv

@@ -0,0 +1,2 @@
+another  country or territory sign a formel agreement on behalf? of their communities endorsing a
+soster citues international

example_data/combined_case_notes.csv

@@ -0,0 +1,19 @@
+Date,Social Worker,Client,Case Note
+"January 3, 2023",Jane Smith,Alex D.,"Met with Alex at school following reports of increased absences and declining grades. Alex appeared sullen and avoided eye contact. When prompted about school, Alex expressed feelings of isolation and stated, ""No one gets me."" Scheduled a follow-up meeting to further explore these feelings."
+"January 17, 2023",Jane Smith,Alex D.,"Met with Alex at the community center. Alex displayed sudden outbursts of anger when discussing home life, particularly in relation to a new stepfather. Alex mentioned occasional substance use, but did not specify which substances. Recommended a comprehensive assessment."
+"February 5, 2023",Jane Smith,Alex D.,Home visit conducted. Alex's mother reported frequent arguments at home. She expressed concerns about Alex's new group of friends and late-night outings. Noted potential signs of substance abuse. Suggested family counseling.
+"February 21, 2023",Jane Smith,Alex D.,"Met with Alex alone at my office. Alex appeared more agitated than in previous meetings. There were visible signs of self-harm on Alex's arms. When questioned, Alex became defensive. Immediate referral made to a mental health professional."
+"March 10, 2023",Jane Smith,Alex D.,Attended joint session with Alex and a therapist. Alex shared feelings of hopelessness and admitted to occasional thoughts of self-harm. Therapist recommended a comprehensive mental health evaluation and ongoing therapy.
+"March 25, 2023",Jane Smith,Alex D.,"Received a call from Alex's school about a physical altercation with another student. Met with Alex, who displayed high levels of frustration and admitted to the use of alcohol. Discussed the importance of seeking help and finding positive coping mechanisms. Recommended enrollment in an anger management program."
+"April 15, 2023",Jane Smith,Alex D.,Met with Alex and mother to discuss progress. Alex's mother expressed concerns about Alex's increasing aggression at home. Alex acknowledged the issues but blamed others for provoking the behavior. It was decided that a more intensive intervention may be needed.
+"April 30, 2023",Jane Smith,Alex D.,"Met with Alex and a psychiatrist. Psychiatrist diagnosed Alex with Oppositional Defiant Disorder (ODD) and co-morbid substance use disorder. A treatment plan was discussed, including medication, therapy, and family counseling."
+"May 20, 2023",Jane Smith,Alex D.,"Met with Alex to discuss progress. Alex has started attending group therapy and has shown slight improvements in behavior. Still, concerns remain about substance use. Discussed potential for a short-term residential treatment program."
+"January 3, 2023",Jane Smith,Jamie L.,"Met with Jamie at school after receiving reports of consistent tardiness and decreased participation in class. Jamie appeared withdrawn and exhibited signs of sadness. When asked about feelings, Jamie expressed feeling ""empty"" and ""hopeless"" at times. Scheduled a follow-up meeting to further explore these feelings."
+"January 17, 2023",Jane Smith,Jamie L.,"Met with Jamie at the community center. Jamie shared feelings of low self-worth, mentioning that it's hard to find motivation for daily tasks. Discussed potential triggers and learned about recent family financial struggles. Recommended counseling and possible group therapy for peer support."
+"February 5, 2023",Jane Smith,Jamie L.,Home visit conducted. Jamie's parents shared concerns about Jamie's increasing withdrawal from family activities and lack of interest in hobbies. Parents mentioned that Jamie spends a lot of time alone in the room. Suggested family therapy to open communication channels.
+"February 21, 2023",Jane Smith,Jamie L.,Met with Jamie in my office. Jamie opened up about feelings of isolation and mentioned difficulty sleeping. No signs of self-harm or suicidal ideation were noted. Recommended a comprehensive mental health assessment to better understand the depth of the depression.
+"March 10, 2023",Jane Smith,Jamie L.,"Attended a joint session with Jamie and a therapist. The therapist noted signs of moderate depression. Together, we discussed coping strategies and potential interventions. Jamie showed interest in art therapy."
+"March 25, 2023",Jane Smith,Jamie L.,"Received feedback from Jamie's school that academic performance has slightly improved. However, social interactions remain limited. Encouraged Jamie to join school clubs or groups to foster connection."
+"April 15, 2023",Jane Smith,Jamie L.,"Met with Jamie and parents to discuss progress. Parents have observed slight improvements in mood on some days, but overall, Jamie still appears to struggle. It was decided to explore medication as a potential aid alongside therapy."
+"April 30, 2023",Jane Smith,Jamie L.,Met with Jamie and a psychiatrist. The psychiatrist diagnosed Jamie with Major Depressive Disorder (MDD) and suggested considering antidepressant medication. Discussed the potential benefits and side effects. Jamie and parents will think it over.
+"May 20, 2023",Jane Smith,Jamie L.,"Jamie has started on a low dose of an antidepressant. Initial feedback is positive, with some improvement in mood and energy levels. Will continue monitoring and adjusting as necessary."

example_data/doubled_output_joined.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eeac353164447c2aa429196e1a6ffae4c095d7171e63c2d1cd1966fdf32d1ed
+size 1274719

example_data/example_of_emails_sent_to_a_professor_before_applying.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0cd82b5b5826b851ca0e7c102d2d4d27580f7a90de4211a33178a6664d008d
+size 8848

example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_ocr_output.csv

@@ -0,0 +1,277 @@
+page,text,left,top,width,height
+1,Partnership Agreement,0.516078,0.027879,0.440784,0.032424
+1,SisterCities,0.169804,0.033333,0.238431,0.028182
+1,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788
+1,Toolkit,0.830588,0.07303,0.126667,0.025152
+1,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303
+1,Types of Affiliations,0.117255,0.157576,0.241961,0.02
+1,Sister City Relationship,0.117647,0.187273,0.196863,0.013939
+1,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636
+1,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939
+1,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636
+1,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939
+1,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636
+1,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636
+1,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636
+1,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636
+1,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636
+1,Friendship City,0.118039,0.372121,0.127059,0.013939
+1,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636
+1,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242
+1,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636
+1,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636
+1,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636
+1,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333
+1,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636
+1,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636
+1,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636
+1,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333
+1,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727
+1,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636
+1,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333
+1,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939
+1,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242
+1,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636
+1,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636
+1,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939
+1,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939
+1,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636
+1,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939
+1,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242
+1,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333
+1,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939
+1,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636
+1,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333
+1,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636
+1,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636
+2,Partnership Agreement,0.516078,0.027879,0.440784,0.032424
+2,SisterCities,0.169804,0.033333,0.238824,0.028182
+2,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788
+2,Toolkit,0.83098,0.072727,0.127059,0.025455
+2,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303
+2,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333
+2,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333
+2,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333
+2,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636
+2,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333
+2,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333
+2,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333
+2,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333
+2,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636
+2,General Guidelines,0.118039,0.295152,0.231765,0.016061
+2,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636
+2,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636
+2,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636
+2,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636
+2,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636
+2,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333
+2,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636
+2,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636
+2,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939
+2,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636
+2,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636
+2,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636
+2,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939
+2,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333
+2,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636
+2,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636
+2,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939
+2,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636
+2,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939
+2,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636
+2,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333
+2,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636
+2,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636
+2,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636
+2,with very specific tasks.,0.176078,0.750606,0.190196,0.013333
+2,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636
+2,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333
+2,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939
+2,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636
+2,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939
+2,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636
+3,Partnership Agreement,0.516078,0.027879,0.441176,0.032121
+3,SisterCities,0.169804,0.033333,0.239216,0.028182
+3,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788
+3,Toolkit,0.83098,0.07303,0.126667,0.025152
+3,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303
+3,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333
+3,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333
+3,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636
+3,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636
+3,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636
+3,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333
+3,and cooperation.,0.176471,0.25697,0.13451,0.013333
+3,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333
+3,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636
+3,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333
+3,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636
+3,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636
+3,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636
+3,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636
+3,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333
+3,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636
+3,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333
+3,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636
+3,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636
+3,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636
+3,for their records.,0.176078,0.550606,0.131373,0.010606
+3,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636
+3,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636
+3,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333
+3,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939
+3,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636
+3,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242
+3,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515
+3,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939
+3,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636
+3,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636
+3,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636
+3,sending it to our Membership Director at [email protected] or contacting us at (202),0.117647,0.782727,0.732157,0.013636
+3,347-8630.,0.117647,0.799394,0.080392,0.010303
+4,Partnership Agreement,0.516471,0.027879,0.440784,0.032727
+4,SisterCities,0.169412,0.033333,0.239608,0.028485
+4,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091
+4,Toolkit,0.830588,0.072727,0.127843,0.025758
+4,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333
+4,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394
+4,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667
+4,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727
+4,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121
+4,BETWEEN,0.454902,0.413636,0.110588,0.011212
+4,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939
+4,AND,0.487843,0.452727,0.048235,0.011212
+4,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848
+4,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303
+4,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727
+4,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727
+4,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424
+4,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424
+4,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303
+4,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121
+4,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424
+4,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121
+4,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758
+4,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303
+4,relationship became effective.,0.221569,0.705152,0.217647,0.012424
+4,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303
+4,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727
+4,A,0.344314,0.768485,0.084706,0.030303
+4,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909
+4,Lee P.Brown,0.729412,0.806364,0.118824,0.010303
+4,Mayor of Houston,0.704706,0.823333,0.166667,0.012424
+4,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727
+4,&Town Planning,0.324314,0.841212,0.155686,0.012424
+5,Partnership Agreement,0.516078,0.027879,0.441176,0.032424
+5,SisterCities,0.169412,0.033333,0.239608,0.028485
+5,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091
+5,Toolkit,0.83098,0.072727,0.127059,0.025758
+5,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333
+5,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697
+5,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697
+5,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303
+5,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818
+5,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333
+5,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242
+5,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636
+5,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152
+5,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455
+5,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152
+5,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152
+5,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455
+5,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848
+5,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939
+5,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545
+5,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848
+5,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152
+5,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242
+5,the cities;,0.22902,0.624545,0.076471,0.012424
+5,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152
+5,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636
+5,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636
+5,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848
+5,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545
+5,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152
+5,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545
+5,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909
+5,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636
+5,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091
+5,Mayor,0.311373,0.894848,0.053333,0.012727
+5,New York City,0.287843,0.909091,0.121176,0.013333
+5,London,0.701961,0.909091,0.061569,0.010606
+6,Partnership Agreement,0.515686,0.027576,0.441961,0.03303
+6,SisterCities,0.169412,0.03303,0.24,0.028182
+6,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091
+6,Toolkit,0.83098,0.072727,0.127451,0.025758
+6,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333
+6,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364
+6,City of Long Beach,0.388627,0.196667,0.476471,0.066364
+6,California,0.551373,0.257273,0.136471,0.033333
+6,Sister City Agreement,0.321961,0.305455,0.378431,0.035152
+6,between the,0.464706,0.352727,0.084314,0.009697
+6,City of Long Beach,0.38,0.378485,0.252549,0.01697
+6,"California, USA",0.4,0.397576,0.21098,0.016061
+6,and the,0.48,0.415152,0.053333,0.009091
+6,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697
+6,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152
+6,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121
+6,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303
+6,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121
+6,purposes:,0.216863,0.516061,0.058039,0.009394
+6,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424
+6,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424
+6,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424
+6,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121
+6,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121
+6,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121
+6,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727
+6,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697
+6,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727
+6,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424
+6,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121
+6,STATE OFFICE,0.276471,0.713636,0.050588,0.048788
+6,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636
+6,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636
+6,"California, USA",0.582745,0.765758,0.125098,0.01303
+6,10.2aulus,0.490588,0.771818,0.220392,0.062424
+6,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333
+6,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636
+6,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818
+6,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303
+7,Partnership Agreement,0.516078,0.027879,0.441176,0.032424
+7,SisterCities,0.169412,0.03303,0.24,0.028485
+7,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091
+7,Toolkit,0.83098,0.072727,0.127451,0.025758
+7,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333
+7,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939
+7,adopted by,0.2,0.213333,0.080392,0.013636
+7,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424
+7,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515
+7,and,0.199608,0.260909,0.026275,0.010606
+7,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212
+7,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212
+7,ON,0.551765,0.298182,0.026667,0.011515
+7,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848
+7,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152
+7,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455
+7,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848
+7,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242
+7,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848
+7,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242
+7,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152
+7,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242
+7,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242
+7,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455
+7,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636
+7,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455
+7,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606
+7,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758
+7,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242
+7,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848
+7,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152
+7,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636
+7,3h.5.,0.593725,0.750606,0.218039,0.06303
+7,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818
+7,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606
+7,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303
+7,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606

example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_review_file.csv

@@ -0,0 +1,77 @@
+image,page,label,color,xmin,ymin,xmax,ymax,id,text
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.598431,0.524545,0.63098,0.535455,EG3nykuwvxbk,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.820392,0.798485,0.854118,0.809394,jy1R42e6phNz,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.433333,0.863333,0.46549,0.873939,9sbrsroLfZy0,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_1.png,2,ADDRESS,"(0, 0, 0)",0.354118,0.188788,0.386275,0.199697,k7bWBsQQchJZ,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_1.png,2,ADDRESS,"(0, 0, 0)",0.780392,0.204848,0.812941,0.215758,peo6UqIxrjmR,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,EMAIL,"(0, 0, 0)",0.447843,0.78303,0.648627,0.796667,DIfz0LenOtQv,[email protected]
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,PHONE,"(0, 0, 0)",0.809804,0.78303,0.850196,0.796667,odJdySe9XrAn,(202)
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,PHONE,"(0, 0, 0)",0.117647,0.799394,0.198431,0.809697,iURSkUM7BbUG,347-8630
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.637647,0.432727,0.712941,0.44697,fRxAD9qm856s,U. A.E
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.489412,0.43303,0.614902,0.444545,qzRFPlNbslpH,ABU DHABI
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.385882,0.472121,0.593725,0.486364,v1uLbGsofN1f,"HOUSTON, TEXAS"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.392549,0.539697,0.573725,0.549394,MvbPQiHvSdL7,United States of America
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.539216,0.553333,0.635686,0.563333,05U3cgj5w9PY,United States
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.534902,0.594242,0.615294,0.603939,uHMikyBlMq5f,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.651373,0.594242,0.717255,0.605455,XNUE0GopIBaf,Houston
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.221569,0.65,0.301176,0.659697,6FjbNu2CGA9n,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.337647,0.65,0.404314,0.660606,Yvmm2225ityu,Houston
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,HANDWRITING,"(0, 0, 0)",0.344314,0.768485,0.42902,0.798788,EwTcqq7PENU8,A
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.245882,0.806364,0.612549,0.817576,Mj4gqwbgsZWp,Sheikh Mohammed bin Butti AI Hamed
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.52,0.806364,0.612549,0.81697,RXYOVgLwq8Ke,AI Hamed
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.729412,0.806364,0.848235,0.816667,REPZhwFWGoTc,Lee P.Brown
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.245882,0.806667,0.51451,0.817576,rFdxMRFRWLRJ,Sheikh Mohammed bin Butti
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.366667,0.823939,0.465098,0.834242,5iYCxRGdPG1i,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.577647,0.262121,0.68,0.271515,3ZR43H3yYNdy,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.461176,0.29303,0.555294,0.303333,WNoitmR9A6lu,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.461176,0.29303,0.658039,0.303333,HjrhxMQhovlF,NEW YORK N.Y. 10007
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.563137,0.29303,0.658039,0.302121,nPN7g7UcnX4u,N.Y. 10007
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.314118,0.356667,0.42549,0.367576,ZoJf29CB3Wrq,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.655294,0.480909,0.718431,0.491515,iezAqmD2ilnb,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.708627,0.639394,0.837255,0.652727,tWAuJEQVpfhi,New York City
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.60902,0.64,0.67098,0.650606,NaW3mmmlhMW9,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.667059,0.702727,0.751373,0.713636,pgMiwuMiBp8B,New York
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.198824,0.720303,0.261569,0.731212,fPvElSFZFRoL,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,HANDWRITING,"(0, 0, 0)",0.178824,0.795455,0.281961,0.896364,DfniF7P2bXAw,Thedder
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME,"(0, 0, 0)",0.178824,0.795455,0.423529,0.896364,QwnWsAeslO5f,Thedder Rudolph W. Giuliani
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME - ADDRESS,"(0, 0, 0)",0.672157,0.877576,0.80549,0.891212,Vdp95SShYOEO,Ken Livingstone
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.710196,0.877576,0.80549,0.891212,H5DGqsucPAjc,Livingstone
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME,"(0, 0, 0)",0.672157,0.877879,0.705098,0.888182,qotGtnMbhAJr,Ken
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.287843,0.909091,0.40902,0.922727,sFX0tNJJzpE5,New York City
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.701961,0.909091,0.763922,0.919697,2xFbVTbxiOhC,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.55451,0.203636,0.86549,0.258485,Nfe3WTBembGQ,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.551373,0.257273,0.687843,0.290606,kndQY5X4itc8,California
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.558824,0.397879,0.611373,0.410303,B5vq8yhWLeOg,USA
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425882,0.429091,0.691373,0.441818,OtNgqUkoEaZb,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.347451,0.447879,0.665098,0.46303,Q52VzBx2SWNF,"Ecuador, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.724314,0.482121,0.798431,0.493939,O7gd9ywvKsKh,"Long Beach,"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425098,0.49303,0.506275,0.502727,DzYr3xrM8Tvv,San Pablo de
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425098,0.49303,0.715294,0.50303,iZ0knpQD54UU,"San Pablo de Manta, Ecundor, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.509804,0.49303,0.715294,0.50303,pZnYGzr7Pwsl,"Manta, Ecundor, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.217647,0.493333,0.321961,0.504242,r7Aar8FNQF6D,"California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.471765,0.543636,0.596863,0.553939,zg9uBDlSuuA1,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.295294,0.544242,0.36549,0.556061,A0OY6RjMEocW,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.563137,0.655152,0.748627,0.667576,HQlTdEUhOCgI,"Long Beach, California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.463529,0.665758,0.557255,0.674848,bCN9b7kJw0Ik,South America
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.277647,0.666061,0.403529,0.676061,qffN3bDgWRMk,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.587451,0.736667,0.709804,0.750303,eqMENFw5mbnL,Beverly 0 Neill
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.663137,0.751212,0.753333,0.764545,POqPQVBCES8h,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.582745,0.765758,0.708235,0.779091,mjrjsSMOxwaY,"California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,HANDWRITING,"(0, 0, 0)",0.490588,0.771818,0.71098,0.834242,xL8dSawihWuY,10.2aulus
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,NAME,"(0, 0, 0)",0.559608,0.825152,0.769804,0.838485,fHyvwmbOgLMJ,Jorge O. Zambrano Cedeño
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.624314,0.839394,0.782745,0.850303,zGhskyehufSv,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.551765,0.854242,0.74,0.866061,dSPXmtb8M4nt,"Ecuador, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.556471,0.215152,0.731765,0.226667,BEhuvaI5BVaR,RICHARD M. DALEY
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.563137,0.261212,0.725098,0.272424,coo8KK7q6A72,ZHANG RONGMAO
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.566275,0.273636,0.666275,0.285152,0P9rVSbeNdB4,SHENYANG
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.526667,0.380303,0.588235,0.394242,1GDArufutI5y,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.628235,0.380606,0.702353,0.394242,QyD751r4fCU1,Shenyang
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.736863,0.411515,0.868235,0.424545,rntIekANI8BO,Zhang Rongmao
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.199216,0.411818,0.34,0.424848,96TaHazXGIM7,Richard M. Daley
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.514902,0.412424,0.580784,0.425758,kbyVj6qhZSPi,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.696471,0.443939,0.774118,0.45697,rJpaMvepsNln,Shenyang
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.353725,0.474545,0.415686,0.489091,PokCVpLQmDki,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.407451,0.554545,0.469804,0.568182,HqVr414KRg59,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,HANDWRITING,"(0, 0, 0)",0.593725,0.750606,0.811765,0.813636,xdawEv0DUH6P,3h.5.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.730196,0.819394,0.876471,0.830606,Gghr7ccN6lS2,ZHANG RONGMAO
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.34,0.821515,0.501176,0.831515,vOMIv1RS5Sag,RICHARD M. DALEY

example_data/example_outputs/example_of_emails_sent_to_a_professor_before_applying_ocr_output_textract.csv

@@ -0,0 +1,40 @@
+page,text,left,top,width,height,line
+1,Example of emails sent to a professor before applying:,0.147059,0.093434,0.426471,0.013889,1
+1,Fwd: Prospective Graduate Student,0.145425,0.128788,0.277778,0.013889,2
+1,"Dr. Kornbluth,",0.147059,0.162879,0.114379,0.012626,3
+1,I am a senior biology major at the University of Notre Dame. I am applying to the CMB,0.147059,0.198232,0.689542,0.013889,4
+1,program and am very interested in your work. After glancing at a few of your recent,0.145425,0.214646,0.660131,0.013889,5
+1,papers and your research summary I find your work with apoptosis very interesting. Will,0.145425,0.232323,0.697712,0.013889,6
+1,"you be taking on new students next year? If I am invited to interview, is there any way",0.145425,0.25,0.683007,0.013889,7
+1,you will be able to meet with me?,0.145425,0.267677,0.264706,0.013889,8
+1,I have worked on several different research projects as an undergraduate in Dr. David R.,0.147059,0.30303,0.69281,0.013889,9
+1,Hyde's lab at the University of Notre Dame. The Hyde lab is interested in the signals that,0.147059,0.320707,0.697712,0.013889,10
+1,initiate Muller glia division post-light damage. My first research project was,0.147059,0.338384,0.598039,0.013889,11
+1,characterizing the role of leukemia inhibitory factor (LIF) in the activation of cell,0.147059,0.354798,0.637255,0.013889,12
+1,proliferation in the undamaged zebrafish retina. I am also working on several,0.145425,0.372475,0.604575,0.013889,13
+1,experiments that are related to a genetic screen that the Hyde lab plans on performing to,0.145425,0.390152,0.689542,0.013889,14
+1,identify mutants in the regeneration pathway--I am developing a neuroD4:EGFP,0.147059,0.407828,0.635621,0.013889,15
+1,transgenic line for use in this screen and I am characterizing the extent of damage and,0.145425,0.425505,0.673203,0.013889,16
+1,"regeneration in sheer zebrafish retinas. Finally, I am characterizing the chx10:EGFP",0.145425,0.443182,0.661765,0.013889,17
+1,transgenic line during retinal development and regeneration.,0.145425,0.459596,0.472222,0.013889,18
+1,Please find my CV attached.,0.145425,0.496212,0.222222,0.013889,19
+1,"Thank you for your time,",0.145425,0.531566,0.196078,0.013889,20
+1,--Lauren Lilley,0.147059,0.566919,0.119281,0.013889,21
+1,"Dr. Poss,",0.145425,0.637626,0.070261,0.012626,22
+1,I am a senior biology major at the University of Notre Dame. I am applying to your,0.145425,0.671717,0.655229,0.013889,23
+1,graduate program and am very interested in your work. After glancing at a few of your,0.145425,0.689394,0.679739,0.013889,24
+1,recent papers and your research summary I find your research greatly coincides with my,0.145425,0.707071,0.69281,0.013889,25
+1,research experiences and interests. Will you be taking on new students next year?,0.145425,0.723485,0.643791,0.015152,26
+1,I have worked on several different research projects as an undergraduate in Dr. David R.,0.145425,0.760101,0.69281,0.013889,27
+1,Hyde's lab at the University of Notre Dame. The Hyde lab is interested in the signals that,0.145425,0.777778,0.699346,0.013889,28
+1,initiate Muller glia division post-light damage. My first research project was,0.145425,0.795455,0.598039,0.013889,29
+1,characterizing the role of leukemia inhibitory factor (LIF) in the activation of cell,0.145425,0.811869,0.638889,0.013889,30
+1,proliferation in the undamaged zebrafish retina. I am also working on several,0.145425,0.829545,0.604575,0.013889,31
+1,experiments that are related to a genetic screen that the Hyde lab plans on performing to,0.145425,0.847222,0.691176,0.013889,32
+1,identify mutants in the regeneration pathway--I am developing a neuroD4:EGFP,0.145425,0.864899,0.635621,0.013889,33
+1,transgenic line for use in this screen and I am characterizing the extent of damage and,0.145425,0.881313,0.673203,0.013889,34
+2,"regeneration in sheer zebrafish retinas. Finally, I am characterizing the chx10:EGFP",0.145425,0.093434,0.661765,0.013889,1
+2,transgenic line during retinal development and regeneration.,0.145425,0.111111,0.472222,0.013889,2
+2,Please find my CV attached.,0.145425,0.146465,0.222222,0.013889,3
+2,"Thank you for your time,",0.145425,0.181818,0.196078,0.013889,4
+2,--Lauren Lilley,0.147059,0.218434,0.119281,0.013889,5

example_data/example_outputs/example_of_emails_sent_to_a_professor_before_applying_ocr_results_with_words_textract.csv

@@ -0,0 +1,432 @@
+page,line,word_text,word_x0,word_y0,word_x1,word_y1,line_text,line_x0,line_y0,line_x1,line_y1
+1,1,Example,0.147059,0.093434,0.215686,0.107323,,,,,
+1,1,of,0.220588,0.093434,0.240196,0.104798,,,,,
+1,1,emails,0.24183,0.093434,0.292484,0.104798,,,,,
+1,1,sent,0.297386,0.094697,0.330065,0.104798,,,,,
+1,1,to,0.334967,0.094697,0.349673,0.104798,,,,,
+1,1,a,0.354575,0.097222,0.362745,0.104798,,,,,
+1,1,professor,0.367647,0.093434,0.441176,0.108586,,,,,
+1,1,before,0.446078,0.093434,0.496732,0.104798,,,,,
+1,1,applying:,0.501634,0.093434,0.573529,0.107323,,,,,
+1,2,Fwd:,0.145425,0.128788,0.184641,0.140152,,,,,
+1,2,Prospective,0.191176,0.128788,0.28268,0.142677,,,,,
+1,2,Graduate,0.287582,0.128788,0.359477,0.140152,,,,,
+1,2,Student,0.364379,0.128788,0.424837,0.140152,,,,,
+1,3,Dr.,0.147059,0.162879,0.171569,0.174242,,,,,
+1,3,"Kornbluth,",0.176471,0.162879,0.261438,0.176768,,,,,
+1,4,I,0.147059,0.198232,0.153595,0.209596,,,,,
+1,4,am,0.158497,0.200758,0.181373,0.209596,,,,,
+1,4,a,0.186275,0.20202,0.194444,0.209596,,,,,
+1,4,senior,0.199346,0.198232,0.248366,0.209596,,,,,
+1,4,biology,0.253268,0.198232,0.312092,0.212121,,,,,
+1,4,major,0.316993,0.198232,0.364379,0.212121,,,,,
+1,4,at,0.367647,0.199495,0.382353,0.209596,,,,,
+1,4,the,0.387255,0.198232,0.411765,0.209596,,,,,
+1,4,University,0.416667,0.198232,0.5,0.212121,,,,,
+1,4,of,0.504902,0.198232,0.522876,0.209596,,,,,
+1,4,Notre,0.52451,0.198232,0.570261,0.209596,,,,,
+1,4,Dame.,0.575163,0.198232,0.625817,0.209596,,,,,
+1,4,I,0.632353,0.198232,0.637255,0.209596,,,,,
+1,4,am,0.643791,0.200758,0.666667,0.209596,,,,,
+1,4,applying,0.671569,0.198232,0.740196,0.212121,,,,,
+1,4,to,0.745098,0.199495,0.759804,0.209596,,,,,
+1,4,the,0.764706,0.198232,0.789216,0.209596,,,,,
+1,4,CMB,0.794118,0.198232,0.836601,0.209596,,,,,
+1,5,program,0.145425,0.218434,0.212418,0.229798,,,,,
+1,5,and,0.21732,0.215909,0.245098,0.227273,,,,,
+1,5,am,0.25,0.218434,0.27451,0.227273,,,,,
+1,5,very,0.279412,0.218434,0.313725,0.229798,,,,,
+1,5,interested,0.320261,0.214646,0.395425,0.22601,,,,,
+1,5,in,0.400327,0.214646,0.416667,0.22601,,,,,
+1,5,your,0.419935,0.218434,0.457516,0.229798,,,,,
+1,5,work.,0.460784,0.214646,0.506536,0.227273,,,,,
+1,5,After,0.511438,0.214646,0.553922,0.227273,,,,,
+1,5,glancing,0.55719,0.215909,0.625817,0.229798,,,,,
+1,5,at,0.630719,0.217172,0.645425,0.227273,,,,,
+1,5,a,0.650327,0.218434,0.658497,0.227273,,,,,
+1,5,few,0.663399,0.214646,0.69281,0.22601,,,,,
+1,5,of,0.697712,0.214646,0.715686,0.227273,,,,,
+1,5,your,0.718954,0.218434,0.754902,0.229798,,,,,
+1,5,recent,0.759804,0.217172,0.80719,0.22601,,,,,
+1,6,papers,0.145425,0.236111,0.197712,0.247475,,,,,
+1,6,and,0.202614,0.232323,0.230392,0.243687,,,,,
+1,6,your,0.235294,0.236111,0.271242,0.247475,,,,,
+1,6,research,0.276144,0.232323,0.341503,0.243687,,,,,
+1,6,summary,0.346405,0.236111,0.419935,0.247475,,,,,
+1,6,I,0.424837,0.232323,0.431373,0.243687,,,,,
+1,6,find,0.436275,0.232323,0.46732,0.243687,,,,,
+1,6,your,0.472222,0.236111,0.50817,0.247475,,,,,
+1,6,work,0.513072,0.232323,0.553922,0.243687,,,,,
+1,6,with,0.558824,0.232323,0.593137,0.243687,,,,,
+1,6,apoptosis,0.598039,0.233586,0.671569,0.247475,,,,,
+1,6,very,0.678105,0.236111,0.712418,0.247475,,,,,
+1,6,interesting.,0.71732,0.232323,0.803922,0.247475,,,,,
+1,6,Will,0.810458,0.232323,0.844771,0.243687,,,,,
+1,7,you,0.145425,0.253788,0.174837,0.263889,,,,,
+1,7,be,0.179739,0.25,0.199346,0.261364,,,,,
+1,7,taking,0.204248,0.25,0.253268,0.265152,,,,,
+1,7,on,0.25817,0.253788,0.277778,0.261364,,,,,
+1,7,new,0.28268,0.253788,0.315359,0.261364,,,,,
+1,7,students,0.320261,0.25,0.383987,0.261364,,,,,
+1,7,next,0.388889,0.251263,0.423203,0.261364,,,,,
+1,7,year?,0.428105,0.25,0.470588,0.263889,,,,,
+1,7,If,0.480392,0.25,0.495098,0.261364,,,,,
+1,7,I,0.498366,0.25,0.504902,0.261364,,,,,
+1,7,am,0.509804,0.253788,0.534314,0.261364,,,,,
+1,7,invited,0.539216,0.25,0.593137,0.261364,,,,,
+1,7,to,0.598039,0.251263,0.612745,0.261364,,,,,
+1,7,"interview,",0.617647,0.25,0.696078,0.263889,,,,,
+1,7,is,0.702614,0.25,0.714052,0.261364,,,,,
+1,7,there,0.718954,0.25,0.759804,0.261364,,,,,
+1,7,any,0.763072,0.253788,0.792484,0.263889,,,,,
+1,7,way,0.797386,0.253788,0.830065,0.263889,,,,,
+1,8,you,0.145425,0.271465,0.176471,0.281566,,,,,
+1,8,will,0.179739,0.267677,0.210784,0.27904,,,,,
+1,8,be,0.215686,0.267677,0.235294,0.27904,,,,,
+1,8,able,0.238562,0.267677,0.272876,0.27904,,,,,
+1,8,to,0.276144,0.268939,0.292484,0.27904,,,,,
+1,8,meet,0.297386,0.268939,0.334967,0.27904,,,,,
+1,8,with,0.339869,0.267677,0.375817,0.27904,,,,,
+1,8,me?,0.380719,0.267677,0.411765,0.27904,,,,,
+1,9,I,0.147059,0.30303,0.151961,0.314394,,,,,
+1,9,have,0.156863,0.30303,0.194444,0.314394,,,,,
+1,9,worked,0.199346,0.30303,0.25817,0.314394,,,,,
+1,9,on,0.263072,0.306818,0.28268,0.314394,,,,,
+1,9,several,0.287582,0.30303,0.343137,0.314394,,,,,
+1,9,different,0.348039,0.30303,0.416667,0.314394,,,,,
+1,9,research,0.419935,0.30303,0.485294,0.314394,,,,,
+1,9,projects,0.490196,0.30303,0.552288,0.318182,,,,,
+1,9,as,0.558824,0.306818,0.573529,0.314394,,,,,
+1,9,an,0.580065,0.306818,0.598039,0.314394,,,,,
+1,9,undergraduate,0.602941,0.30303,0.714052,0.318182,,,,,
+1,9,in,0.718954,0.30303,0.735294,0.314394,,,,,
+1,9,Dr.,0.740196,0.30303,0.764706,0.314394,,,,,
+1,9,David,0.769608,0.30303,0.816993,0.314394,,,,,
+1,9,R.,0.823529,0.30303,0.839869,0.314394,,,,,
+1,10,Hyde's,0.147059,0.320707,0.199346,0.334596,,,,,
+1,10,lab,0.204248,0.320707,0.228758,0.332071,,,,,
+1,10,at,0.23366,0.32197,0.248366,0.332071,,,,,
+1,10,the,0.251634,0.320707,0.276144,0.332071,,,,,
+1,10,University,0.281046,0.320707,0.364379,0.334596,,,,,
+1,10,of,0.369281,0.320707,0.387255,0.332071,,,,,
+1,10,Notre,0.390523,0.320707,0.434641,0.332071,,,,,
+1,10,Dame.,0.439542,0.320707,0.490196,0.332071,,,,,
+1,10,The,0.496732,0.320707,0.527778,0.332071,,,,,
+1,10,Hyde,0.53268,0.320707,0.573529,0.334596,,,,,
+1,10,lab,0.580065,0.320707,0.602941,0.332071,,,,,
+1,10,is,0.607843,0.320707,0.620915,0.332071,,,,,
+1,10,interested,0.625817,0.320707,0.702614,0.332071,,,,,
+1,10,in,0.707516,0.320707,0.722222,0.332071,,,,,
+1,10,the,0.727124,0.320707,0.751634,0.332071,,,,,
+1,10,signals,0.756536,0.320707,0.810458,0.334596,,,,,
+1,10,that,0.815359,0.320707,0.844771,0.332071,,,,,
+1,11,initiate,0.147059,0.338384,0.20098,0.349747,,,,,
+1,11,Muller,0.205882,0.338384,0.259804,0.349747,,,,,
+1,11,glia,0.264706,0.338384,0.292484,0.352273,,,,,
+1,11,division,0.297386,0.338384,0.361111,0.349747,,,,,
+1,11,post-light,0.366013,0.338384,0.44281,0.352273,,,,,
+1,11,damage.,0.446078,0.338384,0.511438,0.352273,,,,,
+1,11,My,0.51634,0.338384,0.544118,0.352273,,,,,
+1,11,first,0.54902,0.338384,0.581699,0.349747,,,,,
+1,11,research,0.584967,0.338384,0.650327,0.349747,,,,,
+1,11,project,0.655229,0.338384,0.710784,0.353535,,,,,
+1,11,was,0.715686,0.340909,0.745098,0.349747,,,,,
+1,12,characterizing,0.147059,0.354798,0.256536,0.369949,,,,,
+1,12,the,0.261438,0.356061,0.285948,0.367424,,,,,
+1,12,role,0.29085,0.356061,0.321895,0.367424,,,,,
+1,12,of,0.326797,0.356061,0.344771,0.367424,,,,,
+1,12,leukemia,0.348039,0.356061,0.419935,0.367424,,,,,
+1,12,inhibitory,0.424837,0.354798,0.501634,0.369949,,,,,
+1,12,factor,0.506536,0.356061,0.553922,0.367424,,,,,
+1,12,(LIF),0.55719,0.354798,0.599673,0.369949,,,,,
+1,12,in,0.604575,0.356061,0.620915,0.367424,,,,,
+1,12,the,0.624183,0.356061,0.648693,0.366162,,,,,
+1,12,activation,0.653595,0.356061,0.732026,0.367424,,,,,
+1,12,of,0.735294,0.354798,0.754902,0.367424,,,,,
+1,12,cell,0.756536,0.356061,0.785948,0.367424,,,,,
+1,13,proliferation,0.145425,0.372475,0.243464,0.387626,,,,,
+1,13,in,0.25,0.373737,0.264706,0.383838,,,,,
+1,13,the,0.269608,0.373737,0.292484,0.383838,,,,,
+1,13,undamaged,0.297386,0.372475,0.388889,0.387626,,,,,
+1,13,zebrafish,0.393791,0.372475,0.465686,0.383838,,,,,
+1,13,retina.,0.470588,0.373737,0.519608,0.383838,,,,,
+1,13,I,0.52451,0.373737,0.531046,0.383838,,,,,
+1,13,am,0.535948,0.376263,0.560458,0.383838,,,,,
+1,13,also,0.565359,0.372475,0.596405,0.383838,,,,,
+1,13,working,0.601307,0.372475,0.666667,0.387626,,,,,
+1,13,on,0.671569,0.376263,0.691176,0.385101,,,,,
+1,13,several,0.696078,0.373737,0.751634,0.383838,,,,,
+1,14,experiments,0.145425,0.390152,0.24183,0.405303,,,,,
+1,14,that,0.246732,0.390152,0.276144,0.401515,,,,,
+1,14,are,0.281046,0.393939,0.305556,0.401515,,,,,
+1,14,related,0.308824,0.390152,0.362745,0.401515,,,,,
+1,14,to,0.367647,0.392677,0.383987,0.401515,,,,,
+1,14,a,0.388889,0.393939,0.397059,0.401515,,,,,
+1,14,genetic,0.401961,0.390152,0.45915,0.405303,,,,,
+1,14,screen,0.464052,0.393939,0.514706,0.401515,,,,,
+1,14,that,0.517974,0.390152,0.547386,0.401515,,,,,
+1,14,the,0.552288,0.390152,0.576797,0.401515,,,,,
+1,14,Hyde,0.581699,0.390152,0.624183,0.405303,,,,,
+1,14,lab,0.629085,0.390152,0.653595,0.401515,,,,,
+1,14,plans,0.658497,0.390152,0.699346,0.405303,,,,,
+1,14,on,0.704248,0.393939,0.723856,0.401515,,,,,
+1,14,performing,0.728758,0.390152,0.816993,0.405303,,,,,
+1,14,to,0.821895,0.391414,0.836601,0.401515,,,,,
+1,15,identify,0.147059,0.407828,0.207516,0.421717,,,,,
+1,15,mutants,0.212418,0.409091,0.272876,0.419192,,,,,
+1,15,in,0.279412,0.407828,0.294118,0.419192,,,,,
+1,15,the,0.29902,0.407828,0.323529,0.419192,,,,,
+1,15,regeneration,0.328431,0.407828,0.426471,0.42298,,,,,
+1,15,pathway--I,0.429739,0.407828,0.51634,0.42298,,,,,
+1,15,am,0.522876,0.411616,0.545752,0.419192,,,,,
+1,15,developing,0.550654,0.407828,0.638889,0.42298,,,,,
+1,15,a,0.643791,0.411616,0.651961,0.419192,,,,,
+1,15,neuroD4:EGFP,0.656863,0.407828,0.78268,0.419192,,,,,
+1,16,transgenic,0.145425,0.425505,0.227124,0.439394,,,,,
+1,16,line,0.232026,0.425505,0.261438,0.436869,,,,,
+1,16,for,0.26634,0.425505,0.289216,0.436869,,,,,
+1,16,use,0.294118,0.42803,0.320261,0.436869,,,,,
+1,16,in,0.325163,0.425505,0.339869,0.436869,,,,,
+1,16,this,0.344771,0.425505,0.372549,0.436869,,,,,
+1,16,screen,0.377451,0.42803,0.428105,0.436869,,,,,
+1,16,and,0.433007,0.425505,0.460784,0.436869,,,,,
+1,16,I,0.46732,0.425505,0.472222,0.436869,,,,,
+1,16,am,0.477124,0.42803,0.501634,0.436869,,,,,
+1,16,characterizing,0.506536,0.425505,0.617647,0.439394,,,,,
+1,16,the,0.622549,0.425505,0.647059,0.436869,,,,,
+1,16,extent,0.651961,0.426768,0.70098,0.436869,,,,,
+1,16,of,0.704248,0.425505,0.722222,0.436869,,,,,
+1,16,damage,0.72549,0.425505,0.787582,0.439394,,,,,
+1,16,and,0.79085,0.425505,0.820261,0.436869,,,,,
+1,17,regeneration,0.145425,0.443182,0.243464,0.457071,,,,,
+1,17,in,0.25,0.443182,0.264706,0.454545,,,,,
+1,17,sheer,0.267974,0.443182,0.312092,0.454545,,,,,
+1,17,zebrafish,0.316993,0.443182,0.388889,0.454545,,,,,
+1,17,retinas.,0.393791,0.443182,0.449346,0.454545,,,,,
+1,17,"Finally,",0.455882,0.443182,0.51634,0.457071,,,,,
+1,17,I,0.521242,0.443182,0.527778,0.454545,,,,,
+1,17,am,0.53268,0.445707,0.55719,0.454545,,,,,
+1,17,characterizing,0.560458,0.443182,0.671569,0.457071,,,,,
+1,17,the,0.676471,0.443182,0.70098,0.454545,,,,,
+1,17,chx10:EGFP,0.705882,0.443182,0.808824,0.454545,,,,,
+1,18,transgenic,0.145425,0.459596,0.227124,0.474747,,,,,
+1,18,line,0.232026,0.459596,0.261438,0.47096,,,,,
+1,18,during,0.26634,0.459596,0.316993,0.474747,,,,,
+1,18,retinal,0.321895,0.459596,0.372549,0.47096,,,,,
+1,18,development,0.377451,0.459596,0.478758,0.474747,,,,,
+1,18,and,0.48366,0.460859,0.511438,0.47096,,,,,
+1,18,regeneration.,0.51634,0.459596,0.619281,0.474747,,,,,
+1,19,Please,0.145425,0.496212,0.196078,0.507576,,,,,
+1,19,find,0.20098,0.496212,0.232026,0.507576,,,,,
+1,19,my,0.236928,0.5,0.263072,0.510101,,,,,
+1,19,CV,0.267974,0.496212,0.295752,0.507576,,,,,
+1,19,attached.,0.29902,0.496212,0.369281,0.507576,,,,,
+1,20,Thank,0.145425,0.531566,0.196078,0.542929,,,,,
+1,20,you,0.20098,0.535354,0.230392,0.546717,,,,,
+1,20,for,0.235294,0.531566,0.25817,0.542929,,,,,
+1,20,your,0.263072,0.535354,0.29902,0.546717,,,,,
+1,20,"time,",0.303922,0.531566,0.343137,0.545455,,,,,
+1,21,--Lauren,0.147059,0.568182,0.215686,0.579545,,,,,
+1,21,Lilley,0.218954,0.566919,0.26634,0.582071,,,,,
+1,22,Dr.,0.145425,0.637626,0.171569,0.64899,,,,,
+1,22,"Poss,",0.176471,0.637626,0.21732,0.651515,,,,,
+1,23,I,0.145425,0.671717,0.151961,0.683081,,,,,
+1,23,am,0.158497,0.675505,0.181373,0.684343,,,,,
+1,23,a,0.186275,0.675505,0.194444,0.684343,,,,,
+1,23,senior,0.199346,0.671717,0.248366,0.683081,,,,,
+1,23,biology,0.253268,0.671717,0.312092,0.686869,,,,,
+1,23,major,0.316993,0.671717,0.364379,0.686869,,,,,
+1,23,at,0.369281,0.674242,0.382353,0.683081,,,,,
+1,23,the,0.387255,0.671717,0.411765,0.684343,,,,,
+1,23,University,0.416667,0.671717,0.498366,0.686869,,,,,
+1,23,of,0.504902,0.671717,0.522876,0.683081,,,,,
+1,23,Notre,0.52451,0.671717,0.570261,0.684343,,,,,
+1,23,Dame.,0.575163,0.671717,0.625817,0.684343,,,,,
+1,23,I,0.630719,0.671717,0.637255,0.683081,,,,,
+1,23,am,0.643791,0.675505,0.666667,0.684343,,,,,
+1,23,applying,0.671569,0.67298,0.740196,0.686869,,,,,
+1,23,to,0.745098,0.67298,0.759804,0.683081,,,,,
+1,23,your,0.764706,0.675505,0.802288,0.686869,,,,,
+1,24,graduate,0.145425,0.689394,0.214052,0.704545,,,,,
+1,24,program,0.218954,0.693182,0.284314,0.703283,,,,,
+1,24,and,0.289216,0.689394,0.318627,0.700758,,,,,
+1,24,am,0.323529,0.693182,0.348039,0.700758,,,,,
+1,24,very,0.351307,0.693182,0.387255,0.703283,,,,,
+1,24,interested,0.392157,0.689394,0.46732,0.700758,,,,,
+1,24,in,0.473856,0.689394,0.488562,0.700758,,,,,
+1,24,your,0.493464,0.693182,0.529412,0.703283,,,,,
+1,24,work.,0.534314,0.689394,0.578431,0.700758,,,,,
+1,24,After,0.583333,0.689394,0.625817,0.700758,,,,,
+1,24,glancing,0.630719,0.689394,0.697712,0.703283,,,,,
+1,24,at,0.702614,0.690657,0.71732,0.700758,,,,,
+1,24,a,0.722222,0.693182,0.730392,0.700758,,,,,
+1,24,few,0.735294,0.689394,0.764706,0.700758,,,,,
+1,24,of,0.769608,0.689394,0.787582,0.700758,,,,,
+1,24,your,0.79085,0.693182,0.826797,0.703283,,,,,
+1,25,recent,0.145425,0.708333,0.194444,0.718434,,,,,
+1,25,papers,0.199346,0.710859,0.25,0.72096,,,,,
+1,25,and,0.254902,0.707071,0.28268,0.718434,,,,,
+1,25,your,0.287582,0.710859,0.325163,0.72096,,,,,
+1,25,research,0.328431,0.707071,0.393791,0.718434,,,,,
+1,25,summary,0.398693,0.709596,0.472222,0.72096,,,,,
+1,25,I,0.477124,0.707071,0.48366,0.718434,,,,,
+1,25,find,0.488562,0.707071,0.519608,0.718434,,,,,
+1,25,your,0.52451,0.710859,0.562092,0.72096,,,,,
+1,25,research,0.565359,0.707071,0.632353,0.718434,,,,,
+1,25,greatly,0.637255,0.707071,0.691176,0.72096,,,,,
+1,25,coincides,0.696078,0.707071,0.769608,0.718434,,,,,
+1,25,with,0.77451,0.707071,0.810458,0.718434,,,,,
+1,25,my,0.813725,0.710859,0.839869,0.72096,,,,,
+1,26,research,0.145425,0.724747,0.210784,0.736111,,,,,
+1,26,experiences,0.21732,0.724747,0.308824,0.738636,,,,,
+1,26,and,0.313725,0.723485,0.341503,0.736111,,,,,
+1,26,interests.,0.346405,0.723485,0.416667,0.736111,,,,,
+1,26,Will,0.426471,0.723485,0.462418,0.736111,,,,,
+1,26,you,0.465686,0.727273,0.496732,0.738636,,,,,
+1,26,be,0.5,0.723485,0.519608,0.736111,,,,,
+1,26,taking,0.52451,0.724747,0.573529,0.738636,,,,,
+1,26,on,0.578431,0.727273,0.598039,0.736111,,,,,
+1,26,new,0.602941,0.727273,0.635621,0.736111,,,,,
+1,26,students,0.640523,0.724747,0.704248,0.736111,,,,,
+1,26,next,0.70915,0.72601,0.745098,0.734848,,,,,
+1,26,year?,0.748366,0.724747,0.79085,0.738636,,,,,
+1,27,I,0.145425,0.760101,0.151961,0.771465,,,,,
+1,27,have,0.156863,0.760101,0.194444,0.771465,,,,,
+1,27,worked,0.199346,0.760101,0.25817,0.771465,,,,,
+1,27,on,0.263072,0.763889,0.28268,0.771465,,,,,
+1,27,several,0.287582,0.760101,0.343137,0.771465,,,,,
+1,27,different,0.348039,0.760101,0.416667,0.771465,,,,,
+1,27,research,0.419935,0.760101,0.485294,0.771465,,,,,
+1,27,projects,0.490196,0.760101,0.552288,0.775253,,,,,
+1,27,as,0.55719,0.763889,0.573529,0.771465,,,,,
+1,27,an,0.578431,0.763889,0.598039,0.771465,,,,,
+1,27,undergraduate,0.602941,0.760101,0.714052,0.775253,,,,,
+1,27,in,0.718954,0.760101,0.735294,0.771465,,,,,
+1,27,Dr.,0.740196,0.760101,0.764706,0.771465,,,,,
+1,27,David,0.769608,0.760101,0.818627,0.771465,,,,,
+1,27,R.,0.823529,0.760101,0.839869,0.771465,,,,,
+1,28,Hyde's,0.145425,0.777778,0.199346,0.791667,,,,,
+1,28,lab,0.204248,0.777778,0.228758,0.789141,,,,,
+1,28,at,0.23366,0.77904,0.248366,0.789141,,,,,
+1,28,the,0.251634,0.777778,0.276144,0.789141,,,,,
+1,28,University,0.281046,0.777778,0.364379,0.791667,,,,,
+1,28,of,0.369281,0.777778,0.387255,0.789141,,,,,
+1,28,Notre,0.390523,0.777778,0.434641,0.789141,,,,,
+1,28,Dame.,0.439542,0.777778,0.490196,0.789141,,,,,
+1,28,The,0.496732,0.777778,0.527778,0.789141,,,,,
+1,28,Hyde,0.53268,0.777778,0.573529,0.791667,,,,,
+1,28,lab,0.580065,0.777778,0.602941,0.789141,,,,,
+1,28,is,0.607843,0.777778,0.620915,0.789141,,,,,
+1,28,interested,0.625817,0.777778,0.702614,0.789141,,,,,
+1,28,in,0.707516,0.777778,0.722222,0.789141,,,,,
+1,28,the,0.727124,0.777778,0.751634,0.789141,,,,,
+1,28,signals,0.756536,0.777778,0.810458,0.791667,,,,,
+1,28,that,0.815359,0.777778,0.846405,0.789141,,,,,
+1,29,initiate,0.145425,0.795455,0.20098,0.806818,,,,,
+1,29,Muller,0.205882,0.795455,0.259804,0.806818,,,,,
+1,29,glia,0.264706,0.795455,0.292484,0.809343,,,,,
+1,29,division,0.297386,0.795455,0.361111,0.806818,,,,,
+1,29,post-light,0.366013,0.795455,0.44281,0.809343,,,,,
+1,29,damage.,0.446078,0.795455,0.511438,0.809343,,,,,
+1,29,My,0.51634,0.795455,0.544118,0.809343,,,,,
+1,29,first,0.54902,0.795455,0.581699,0.806818,,,,,
+1,29,research,0.584967,0.795455,0.651961,0.806818,,,,,
+1,29,project,0.655229,0.795455,0.710784,0.809343,,,,,
+1,29,was,0.715686,0.799242,0.745098,0.806818,,,,,
+1,30,characterizing,0.145425,0.811869,0.25817,0.82702,,,,,
+1,30,the,0.261438,0.811869,0.285948,0.823232,,,,,
+1,30,role,0.29085,0.813131,0.321895,0.823232,,,,,
+1,30,of,0.326797,0.811869,0.344771,0.824495,,,,,
+1,30,leukemia,0.348039,0.811869,0.419935,0.823232,,,,,
+1,30,inhibitory,0.424837,0.811869,0.501634,0.82702,,,,,
+1,30,factor,0.506536,0.811869,0.553922,0.823232,,,,,
+1,30,(LIF),0.55719,0.813131,0.599673,0.82702,,,,,
+1,30,in,0.604575,0.811869,0.620915,0.824495,,,,,
+1,30,the,0.624183,0.811869,0.648693,0.824495,,,,,
+1,30,activation,0.653595,0.813131,0.732026,0.824495,,,,,
+1,30,of,0.735294,0.811869,0.754902,0.824495,,,,,
+1,30,cell,0.756536,0.811869,0.785948,0.824495,,,,,
+1,31,proliferation,0.145425,0.829545,0.245098,0.844697,,,,,
+1,31,in,0.25,0.829545,0.264706,0.840909,,,,,
+1,31,the,0.267974,0.829545,0.292484,0.840909,,,,,
+1,31,undamaged,0.297386,0.830808,0.388889,0.844697,,,,,
+1,31,zebrafish,0.393791,0.829545,0.465686,0.842172,,,,,
+1,31,retina.,0.470588,0.830808,0.519608,0.842172,,,,,
+1,31,I,0.52451,0.830808,0.531046,0.840909,,,,,
+1,31,am,0.535948,0.833333,0.560458,0.842172,,,,,
+1,31,also,0.565359,0.829545,0.596405,0.840909,,,,,
+1,31,working,0.601307,0.830808,0.666667,0.844697,,,,,
+1,31,on,0.671569,0.833333,0.691176,0.840909,,,,,
+1,31,several,0.696078,0.829545,0.751634,0.840909,,,,,
+1,32,experiments,0.145425,0.847222,0.24183,0.862374,,,,,
+1,32,that,0.246732,0.847222,0.276144,0.858586,,,,,
+1,32,are,0.281046,0.85101,0.305556,0.858586,,,,,
+1,32,related,0.308824,0.847222,0.362745,0.858586,,,,,
+1,32,to,0.367647,0.848485,0.383987,0.858586,,,,,
+1,32,a,0.388889,0.85101,0.397059,0.858586,,,,,
+1,32,genetic,0.401961,0.847222,0.45915,0.861111,,,,,
+1,32,screen,0.464052,0.85101,0.514706,0.858586,,,,,
+1,32,that,0.517974,0.847222,0.54902,0.858586,,,,,
+1,32,the,0.552288,0.847222,0.576797,0.858586,,,,,
+1,32,Hyde,0.581699,0.847222,0.624183,0.861111,,,,,
+1,32,lab,0.629085,0.847222,0.653595,0.858586,,,,,
+1,32,plans,0.656863,0.847222,0.699346,0.861111,,,,,
+1,32,on,0.704248,0.85101,0.723856,0.858586,,,,,
+1,32,performing,0.728758,0.847222,0.816993,0.862374,,,,,
+1,32,to,0.821895,0.848485,0.836601,0.858586,,,,,
+1,33,identify,0.145425,0.864899,0.207516,0.878788,,,,,
+1,33,mutants,0.212418,0.866162,0.272876,0.876263,,,,,
+1,33,in,0.279412,0.864899,0.294118,0.876263,,,,,
+1,33,the,0.29902,0.864899,0.323529,0.876263,,,,,
+1,33,regeneration,0.328431,0.864899,0.426471,0.878788,,,,,
+1,33,pathway--I,0.431373,0.864899,0.51634,0.878788,,,,,
+1,33,am,0.522876,0.868687,0.545752,0.876263,,,,,
+1,33,developing,0.550654,0.864899,0.638889,0.878788,,,,,
+1,33,a,0.643791,0.868687,0.651961,0.876263,,,,,
+1,33,neuroD4:EGFP,0.655229,0.864899,0.78268,0.876263,,,,,
+1,34,transgenic,0.145425,0.882576,0.227124,0.896465,,,,,
+1,34,line,0.232026,0.882576,0.261438,0.893939,,,,,
+1,34,for,0.26634,0.881313,0.289216,0.893939,,,,,
+1,34,use,0.294118,0.885101,0.320261,0.893939,,,,,
+1,34,in,0.325163,0.882576,0.339869,0.893939,,,,,
+1,34,this,0.344771,0.882576,0.372549,0.893939,,,,,
+1,34,screen,0.379085,0.885101,0.428105,0.893939,,,,,
+1,34,and,0.433007,0.882576,0.460784,0.893939,,,,,
+1,34,I,0.46732,0.882576,0.472222,0.893939,,,,,
+1,34,am,0.478758,0.885101,0.501634,0.893939,,,,,
+1,34,characterizing,0.506536,0.882576,0.617647,0.896465,,,,,
+1,34,the,0.622549,0.882576,0.647059,0.893939,,,,,
+1,34,extent,0.651961,0.883838,0.699346,0.892677,,,,,
+1,34,of,0.704248,0.882576,0.722222,0.893939,,,,,
+1,34,damage,0.72549,0.882576,0.785948,0.896465,,,,,
+1,34,and,0.79085,0.882576,0.820261,0.893939,,,,,
+2,1,regeneration,0.145425,0.093434,0.243464,0.107323,,,,,
+2,1,in,0.248366,0.093434,0.264706,0.104798,,,,,
+2,1,sheer,0.267974,0.093434,0.312092,0.104798,,,,,
+2,1,zebrafish,0.316993,0.093434,0.387255,0.104798,,,,,
+2,1,retinas.,0.392157,0.093434,0.449346,0.104798,,,,,
+2,1,"Finally,",0.455882,0.093434,0.514706,0.107323,,,,,
+2,1,I,0.521242,0.093434,0.527778,0.104798,,,,,
+2,1,am,0.53268,0.097222,0.555556,0.104798,,,,,
+2,1,characterizing,0.560458,0.093434,0.671569,0.107323,,,,,
+2,1,the,0.676471,0.093434,0.70098,0.104798,,,,,
+2,1,chx10:EGFP,0.705882,0.093434,0.808824,0.104798,,,,,
+2,2,transgenic,0.145425,0.111111,0.227124,0.125,,,,,
+2,2,line,0.232026,0.111111,0.261438,0.122475,,,,,
+2,2,during,0.26634,0.111111,0.316993,0.125,,,,,
+2,2,retinal,0.321895,0.111111,0.372549,0.122475,,,,,
+2,2,development,0.377451,0.111111,0.478758,0.125,,,,,
+2,2,and,0.48366,0.111111,0.511438,0.122475,,,,,
+2,2,regeneration.,0.51634,0.111111,0.617647,0.125,,,,,
+2,3,Please,0.145425,0.146465,0.196078,0.157828,,,,,
+2,3,find,0.20098,0.146465,0.232026,0.157828,,,,,
+2,3,my,0.236928,0.150253,0.263072,0.160354,,,,,
+2,3,CV,0.267974,0.146465,0.295752,0.157828,,,,,
+2,3,attached.,0.29902,0.146465,0.369281,0.157828,,,,,
+2,4,Thank,0.145425,0.183081,0.196078,0.193182,,,,,
+2,4,you,0.20098,0.185606,0.230392,0.19697,,,,,
+2,4,for,0.235294,0.181818,0.25817,0.193182,,,,,
+2,4,your,0.263072,0.185606,0.29902,0.19697,,,,,
+2,4,"time,",0.303922,0.181818,0.343137,0.195707,,,,,
+2,5,--Lauren,0.147059,0.218434,0.215686,0.229798,,,,,
+2,5,Lilley,0.218954,0.218434,0.26634,0.232323,,,,,

example_data/example_outputs/example_of_emails_sent_to_a_professor_before_applying_review_file.csv

@@ -0,0 +1,15 @@
+image,page,label,color,xmin,ymin,xmax,ymax,id,text
+placeholder_image_0.png,1,TITLES,"(0, 0, 0)",0.147059,0.162879,0.171569,0.174242,oJIosRHGyCRn,Dr
+placeholder_image_0.png,1,TITLES - NAME,"(0, 0, 0)",0.147059,0.162879,0.261438,0.176768,5C5tA6mfeL7T,Dr Kornbluth
+placeholder_image_0.png,1,NAME,"(0, 0, 0)",0.176471,0.162879,0.261438,0.176768,UoYN48bc2ry5,Kornbluth
+placeholder_image_0.png,1,TITLES,"(0, 0, 0)",0.740196,0.30303,0.764706,0.314394,cAsjVETPEisV,Dr
+placeholder_image_0.png,1,TITLES - NAME,"(0, 0, 0)",0.740196,0.30303,0.839869,0.314394,yQ5HKn4tfT7L,Dr David R.
+placeholder_image_0.png,1,NAME,"(0, 0, 0)",0.769608,0.30303,0.839869,0.314394,LR8phiOYnLWi,David R.
+placeholder_image_0.png,1,NAME,"(0, 0, 0)",0.218954,0.566919,0.26634,0.582071,X8iObIauqZ9k,Lauren Lilley
+placeholder_image_0.png,1,TITLES,"(0, 0, 0)",0.145425,0.637626,0.171569,0.64899,SvWjK2F7R3un,Dr
+placeholder_image_0.png,1,TITLES - NAME,"(0, 0, 0)",0.145425,0.637626,0.21732,0.651515,zKJFVAOszwdM,Dr Poss
+placeholder_image_0.png,1,NAME,"(0, 0, 0)",0.176471,0.637626,0.21732,0.651515,Iqda7ixkzcmg,Poss
+placeholder_image_0.png,1,TITLES,"(0, 0, 0)",0.740196,0.760101,0.764706,0.771465,TWQD93bGI3B3,Dr
+placeholder_image_0.png,1,TITLES - NAME,"(0, 0, 0)",0.740196,0.760101,0.839869,0.771465,vQuQQwqWjSES,Dr David R.
+placeholder_image_0.png,1,NAME,"(0, 0, 0)",0.769608,0.760101,0.839869,0.771465,f8xf6ORJUSnG,David R.
+placeholder_image_1.png,2,NAME,"(0, 0, 0)",0.218954,0.218434,0.26634,0.232323,N0nje9UiCzZK,Lauren Lilley

example_data/graduate-job-example-cover-letter.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71cc851d41f80dd8b045af32657b76bf85dd8f72d39ae08fa43dc7a78256fe35
+size 77045

example_data/partnership_toolkit_redact_custom_deny_list.csv

@@ -0,0 +1,4 @@
+Sister
+Sister City
+Sister Cities
+Friendship City

example_data/partnership_toolkit_redact_some_pages.csv

@@ -0,0 +1,2 @@
+2
+5

example_data/test_allow_list_graduate.csv

@@ -0,0 +1 @@
+Wilson

example_data/test_allow_list_partnership.csv

@@ -0,0 +1 @@
+[email protected]

lambda_entrypoint.py

@@ -9,7 +9,7 @@ print("Lambda entrypoint loading...")
 
 # Initialize S3 client outside the handler for connection reuse
 s3_client = boto3.client("s3", region_name=os.getenv("AWS_REGION", "eu-west-2"))
-print("S3 client initialized")
+print("S3 client initialised")
 
 # Lambda's only writable directory
 TMP_DIR = "/tmp"
@@ -84,48 +84,90 @@ def lambda_handler(event, context):
         'task': arguments.get('task', 'redact'),
         'input_file': input_file_path,
         'output_dir': OUTPUT_DIR,
-        'language': arguments.get('language', 'en_core_web_sm'),
-        'pii_detector': arguments.get('pii_detector', 'Local Spacy model'), # Default to local
+        'input_dir': INPUT_DIR,
+        'language': arguments.get('language', 'en_core_web_lg'),
+        'pii_detector': arguments.get('pii_detector', 'Local'), # Default to local
+        'username': arguments.get('username', 'lambda_user'),
+        'save_to_user_folders': arguments.get('save_to_user_folders', 'False'),
         'ocr_method': arguments.get('ocr_method', 'Tesseract OCR - all PDF types'),
         'page_min': int(arguments.get('page_min', 0)),
-        'page_max': int(arguments.get('page_max', 999)),
+        'page_max': int(arguments.get('page_max', 0)),
+        'handwrite_signature_extraction': arguments.get('handwrite_signature_checkbox', ['Extract handwriting', 'Extract signatures']),
+        
+        # General arguments
+        'local_redact_entities': arguments.get('local_redact_entities', []),
+        'aws_redact_entities': arguments.get('aws_redact_entities', []),
+        'cost_code': arguments.get('cost_code', ''),
+        'save_logs_to_csv': arguments.get('save_logs_to_csv', 'False'),
+        'save_logs_to_dynamodb': arguments.get('save_logs_to_dynamodb', 'False'),
+        'display_file_names_in_logs': arguments.get('display_file_names_in_logs', 'True'),
+        'upload_logs_to_s3': arguments.get('upload_logs_to_s3', 'False'),
+        's3_logs_prefix': arguments.get('s3_logs_prefix', ''),
+        'do_initial_clean': arguments.get('do_initial_clean', 'False'),
+        
+        # PDF/Image specific arguments
+        'images_dpi': float(arguments.get('images_dpi', 300.0)),
+        'chosen_local_ocr_model': arguments.get('chosen_local_ocr_model', 'tesseract'),
+        'preprocess_local_ocr_images': arguments.get('preprocess_local_ocr_images', 'False'),
         
         # Handle optional files like allow/deny lists
-        'allow_list': None,
-        'deny_list': None,
+        'allow_list_file': arguments.get('allow_list_file', ""),
+        'deny_list_file': arguments.get('deny_list_file', ""),
+        'redact_whole_page_file': arguments.get('redact_whole_page_file', ""),
+        
+        # Tabular/Anonymisation arguments
+        'excel_sheets': arguments.get('excel_sheets', []),
+        'fuzzy_mistakes': int(arguments.get('fuzzy_mistakes', 0)),
+        'match_fuzzy_whole_phrase_bool': arguments.get('match_fuzzy_whole_phrase_bool', 'True'),
         
         # Deduplication specific arguments
         'duplicate_type': arguments.get('duplicate_type', 'pages'),
         'similarity_threshold': float(arguments.get('similarity_threshold', 0.95)),
         'min_word_count': int(arguments.get('min_word_count', 3)),
-        'search_query': arguments.get('search_query'),
+        'min_consecutive_pages': int(arguments.get('min_consecutive_pages', 1)),
+        'greedy_match': arguments.get('greedy_match', 'False'),
+        'combine_pages': arguments.get('combine_pages', 'True'),
+        'search_query': arguments.get('search_query', ""),
         'text_columns': arguments.get('text_columns', []),
+        'remove_duplicate_rows': arguments.get('remove_duplicate_rows', 'True'),
+        'anon_strategy': arguments.get('anon_strategy', 'redact'),
         
-        # Add other arguments from your app.py as needed, using .get() for safety
-        'anon_strat': arguments.get('anon_strat', 'redact'),
-        'columns': arguments.get('columns', []),
-        'aws_access_key': None, # Best practice: use IAM Role instead of keys
+        # Textract specific arguments
+        'textract_action': arguments.get('textract_action', ''),
+        'job_id': arguments.get('job_id', ''),
+        'extract_signatures': arguments.get('extract_signatures', False),
+        'textract_bucket': arguments.get('textract_bucket', ''),
+        'textract_input_prefix': arguments.get('textract_input_prefix', ''),
+        'textract_output_prefix': arguments.get('textract_output_prefix', ''),
+        's3_textract_document_logs_subfolder': arguments.get('s3_textract_document_logs_subfolder', ''),
+        'local_textract_document_logs_subfolder': arguments.get('local_textract_document_logs_subfolder', ''),
+        'poll_interval': int(arguments.get('poll_interval', 30)),
+        'max_poll_attempts': int(arguments.get('max_poll_attempts', 120)),
+        
+        # AWS credentials (use IAM Role instead of keys)
+        'aws_access_key': None,
         'aws_secret_key': None,
-        'aws_region': os.getenv("AWS_REGION", "eu-west-2"),
+        'aws_region': os.getenv("AWS_REGION", ""),
         's3_bucket': bucket_name,
+        
         # Set defaults for boolean flags
-        'prepare_images': True,
-        'compress_redacted_pdf': False,
-        'return_pdf_end_of_redaction': True
+        'prepare_images': arguments.get('prepare_images', True),
+        'compress_redacted_pdf': arguments.get('compress_redacted_pdf', False),
+        'return_pdf_end_of_redaction': arguments.get('return_pdf_end_of_redaction', True)
     }
 
     # Download optional files if they are specified
-    allow_list_key = arguments.get('allow_list')
+    allow_list_key = arguments.get('allow_list_file')
     if allow_list_key:
         allow_list_path = os.path.join(INPUT_DIR, 'allow_list.csv')
         download_file_from_s3(bucket_name, allow_list_key, allow_list_path)
-        cli_args['allow_list'] = allow_list_path
+        cli_args['allow_list_file'] = allow_list_path
         
-    deny_list_key = arguments.get('deny_list')
+    deny_list_key = arguments.get('deny_list_file')
     if deny_list_key:
         deny_list_path = os.path.join(INPUT_DIR, 'deny_list.csv')
         download_file_from_s3(bucket_name, deny_list_key, deny_list_path)
-        cli_args['deny_list'] = deny_list_path
+        cli_args['deny_list_file'] = deny_list_path
 
     # 5. Execute the main application logic
     try:

pyproject.toml

@@ -23,8 +23,8 @@ dependencies = [
     "spacy==3.8.7",
     # Direct URL dependency for spacy model
     "en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0.tar.gz",
-    "gradio==5.45.0",
-    "boto3==1.40.10",
+    "gradio==5.46.1",
+    "boto3==1.40.31",
     "pyarrow==21.0.0",
     "openpyxl==3.1.5",
     "Faker==37.5.3",

requirements.txt

@@ -10,9 +10,9 @@ pandas==2.3.2
 scikit-learn==1.7.1
 spacy==3.8.7
 en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0.tar.gz
-gradio==5.45.0
+gradio==5.46.1
 polars==1.33.1
-boto3==1.40.10
+boto3==1.40.31
 pyarrow==21.0.0
 openpyxl==3.1.5
 Faker==37.5.3

tools/aws_functions.py

@@ -171,7 +171,7 @@ def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=DOCU
             final_out_message_str = "Could not upload files to S3 due to: " + str(e)
             print(final_out_message_str)
     else:
-        final_out_message_str = "App not set to run AWS functions"
+        final_out_message_str = "App config will not AWS functions"
 
     return final_out_message_str
 
@@ -227,7 +227,7 @@ def upload_log_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=
             final_out_message_str = "Could not upload files to S3 due to: " + str(e)
             print(final_out_message_str)
     else:
-        final_out_message_str = "App not set to run AWS functions"
+        final_out_message_str = "App config will not AWS functions"
         print(final_out_message_str)
 
     return final_out_message_str

tools/aws_textract.py

@@ -24,6 +24,8 @@ def analyse_page_with_textract(pdf_page_bytes:object, page_no:int, client:str=""
     '''
     Analyse page with AWS Textract
     '''
+
+    print("handwrite_signature_checkbox in analyse_page_with_textract:", handwrite_signature_checkbox)
     if client == "":
         try:               
             if AWS_ACCESS_KEY and AWS_SECRET_KEY:

tools/cli_usage_logger.py

@@ -0,0 +1,302 @@
+"""
+CLI Usage Logger - A simplified version of the Gradio CSVLogger_custom for CLI usage logging.
+This module provides functionality to log usage data from CLI operations to CSV files and optionally DynamoDB.
+"""
+
+import csv
+import os
+import time
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Any, List, Optional
+import boto3
+import botocore
+from tools.aws_functions import upload_log_file_to_s3
+from tools.config import (
+    USAGE_LOGS_FOLDER, 
+    SAVE_LOGS_TO_CSV, 
+    SAVE_LOGS_TO_DYNAMODB,
+    USAGE_LOG_DYNAMODB_TABLE_NAME,
+    DYNAMODB_USAGE_LOG_HEADERS,
+    CSV_USAGE_LOG_HEADERS,
+    DISPLAY_FILE_NAMES_IN_LOGS,
+    HOST_NAME,
+    AWS_REGION,
+    AWS_ACCESS_KEY,
+    AWS_SECRET_KEY,
+    RUN_AWS_FUNCTIONS,
+    S3_USAGE_LOGS_FOLDER,
+    DOCUMENT_REDACTION_BUCKET
+)
+
+
+class CLIUsageLogger:
+    """
+    A simplified usage logger for CLI operations that mimics the functionality
+    of the Gradio CSVLogger_custom class.
+    """
+    
+    def __init__(self, dataset_file_name: str = "usage_log.csv"):
+        """
+        Initialize the CLI usage logger.
+        
+        Args:
+            dataset_file_name: Name of the CSV file to store logs
+        """
+        self.dataset_file_name = dataset_file_name
+        self.flagging_dir = Path(USAGE_LOGS_FOLDER)
+        self.dataset_filepath = None
+        self.headers = None
+        
+    def setup(self, headers: List[str]):
+        """
+        Setup the logger with the specified headers.
+        
+        Args:
+            headers: List of column headers for the CSV file
+        """
+        self.headers = headers
+        self._create_dataset_file()
+        
+    def _create_dataset_file(self):
+        """Create the dataset CSV file with headers if it doesn't exist."""
+        os.makedirs(self.flagging_dir, exist_ok=True)
+        
+        # Add ID and timestamp to headers (matching custom_csvlogger.py structure)
+        full_headers = self.headers + ["id", "timestamp"]
+        
+        self.dataset_filepath = self.flagging_dir / self.dataset_file_name
+        
+        if not Path(self.dataset_filepath).exists():
+            with open(self.dataset_filepath, "w", newline="", encoding="utf-8") as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerow(full_headers)
+            print(f"Created usage log file at: {self.dataset_filepath}")
+        else:
+            print(f"Using existing usage log file at: {self.dataset_filepath}")
+    
+    def log_usage(
+        self,
+        data: List[Any],
+        save_to_csv: bool = None,
+        save_to_dynamodb: bool = None,
+        save_to_s3: bool = None,
+        s3_bucket: str = None,
+        s3_key_prefix: str = None,
+        dynamodb_table_name: str = None,
+        dynamodb_headers: List[str] = None,
+        replacement_headers: List[str] = None
+    ) -> int:
+        """
+        Log usage data to CSV and optionally DynamoDB and S3.
+        
+        Args:
+            data: List of data values to log
+            save_to_csv: Whether to save to CSV (defaults to config setting)
+            save_to_dynamodb: Whether to save to DynamoDB (defaults to config setting)
+            save_to_s3: Whether to save to S3 (defaults to config setting)
+            s3_bucket: S3 bucket name (defaults to config setting)
+            s3_key_prefix: S3 key prefix (defaults to config setting)
+            dynamodb_table_name: DynamoDB table name (defaults to config setting)
+            dynamodb_headers: DynamoDB headers (defaults to config setting)
+            replacement_headers: Replacement headers for CSV (defaults to config setting)
+            
+        Returns:
+            Number of lines written
+        """
+        # Use config defaults if not specified
+        if save_to_csv is None:
+            save_to_csv = SAVE_LOGS_TO_CSV == 'True'
+        if save_to_dynamodb is None:
+            save_to_dynamodb = SAVE_LOGS_TO_DYNAMODB == 'True'
+        if save_to_s3 is None:
+            save_to_s3 = RUN_AWS_FUNCTIONS == "1" and SAVE_LOGS_TO_CSV == 'True'
+        if s3_bucket is None:
+            s3_bucket = DOCUMENT_REDACTION_BUCKET
+        if s3_key_prefix is None:
+            s3_key_prefix = S3_USAGE_LOGS_FOLDER
+        if dynamodb_table_name is None:
+            dynamodb_table_name = USAGE_LOG_DYNAMODB_TABLE_NAME
+        if dynamodb_headers is None:
+            dynamodb_headers = DYNAMODB_USAGE_LOG_HEADERS
+        if replacement_headers is None:
+            replacement_headers = CSV_USAGE_LOG_HEADERS
+            
+        # Generate unique ID and add timestamp (matching custom_csvlogger.py structure)
+        generated_id = str(uuid.uuid4())
+        timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]  # Correct format for Amazon Athena
+        csv_data = data + [generated_id, timestamp]
+        
+        line_count = 0
+        
+        # Save to CSV
+        if save_to_csv and self.dataset_filepath:
+            try:
+                with open(self.dataset_filepath, "a", newline="", encoding="utf-8-sig") as csvfile:
+                    writer = csv.writer(csvfile)
+                    writer.writerow(csv_data)
+                    line_count = 1
+                print(f"Logged usage data to CSV: {self.dataset_filepath}")
+            except Exception as e:
+                print(f"Error writing to CSV: {e}")
+
+        # Upload to S3 if enabled
+        if save_to_s3 and self.dataset_filepath and s3_bucket and s3_key_prefix:
+            try:
+                # Upload the log file to S3
+                upload_result = upload_log_file_to_s3(
+                    local_file_paths=[str(self.dataset_filepath)],
+                    s3_key=s3_key_prefix,
+                    s3_bucket=s3_bucket,
+                    RUN_AWS_FUNCTIONS=RUN_AWS_FUNCTIONS,
+                    SAVE_LOGS_TO_CSV=SAVE_LOGS_TO_CSV
+                )
+                print(f"S3 upload result: {upload_result}")
+            except Exception as e:
+                print(f"Error uploading log file to S3: {e}")
+        
+        # Save to DynamoDB
+        if save_to_dynamodb and dynamodb_table_name and dynamodb_headers:
+            try:
+                # Initialize DynamoDB client
+                if AWS_ACCESS_KEY and AWS_SECRET_KEY:
+                    dynamodb = boto3.resource(
+                        'dynamodb',
+                        region_name=AWS_REGION,
+                        aws_access_key_id=AWS_ACCESS_KEY,
+                        aws_secret_access_key=AWS_SECRET_KEY
+                    )
+                else:
+                    dynamodb = boto3.resource('dynamodb', region_name=AWS_REGION)
+                
+                table = dynamodb.Table(dynamodb_table_name)
+                
+                # Generate unique ID
+                generated_id = str(uuid.uuid4())
+                
+                # Prepare the DynamoDB item
+                item = {
+                    'id': generated_id,
+                    'timestamp': timestamp,
+                }
+                
+                # Map the headers to values
+                item.update({header: str(value) for header, value in zip(dynamodb_headers, data)})
+                
+                table.put_item(Item=item)
+                print("Successfully uploaded usage log to DynamoDB")
+                
+            except Exception as e:
+                print(f"Could not upload usage log to DynamoDB: {e}")
+        
+        return line_count
+
+
+def create_cli_usage_logger() -> CLIUsageLogger:
+    """
+    Create and setup a CLI usage logger with the standard headers.
+    
+    Returns:
+        Configured CLIUsageLogger instance
+    """
+    # Parse CSV headers from config
+    import json
+    try:
+        headers = json.loads(CSV_USAGE_LOG_HEADERS)
+    except:
+        # Fallback headers if parsing fails
+        headers = [
+            "session_hash_textbox", "doc_full_file_name_textbox", "data_full_file_name_textbox",
+            "actual_time_taken_number", "total_page_count", "textract_query_number",
+            "pii_detection_method", "comprehend_query_number", "cost_code",
+            "textract_handwriting_signature", "host_name_textbox", "text_extraction_method",
+            "is_this_a_textract_api_call", "task"
+        ]
+    
+    logger = CLIUsageLogger()
+    logger.setup(headers)
+    return logger
+
+
+def log_redaction_usage(
+    logger: CLIUsageLogger,
+    session_hash: str,
+    doc_file_name: str,
+    data_file_name: str,
+    time_taken: float,
+    total_pages: int,
+    textract_queries: int,
+    pii_method: str,
+    comprehend_queries: int,
+    cost_code: str,
+    handwriting_signature: str,
+    text_extraction_method: str,
+    is_textract_call: bool,
+    task: str,
+    save_to_dynamodb: bool = None,
+    save_to_s3: bool = None,
+    s3_bucket: str = None,
+    s3_key_prefix: str = None
+):
+    """
+    Log redaction usage data using the provided logger.
+    
+    Args:
+        logger: CLIUsageLogger instance
+        session_hash: Session identifier
+        doc_file_name: Document file name (or placeholder if not displaying names)
+        data_file_name: Data file name (or placeholder if not displaying names)
+        time_taken: Time taken for processing in seconds
+        total_pages: Total number of pages processed
+        textract_queries: Number of Textract API calls made
+        pii_method: PII detection method used
+        comprehend_queries: Number of Comprehend API calls made
+        cost_code: Cost code for the operation
+        handwriting_signature: Handwriting/signature extraction options
+        text_extraction_method: Text extraction method used
+        is_textract_call: Whether this was a Textract API call
+        task: The task performed (redact, deduplicate, textract)
+        save_to_dynamodb: Whether to save to DynamoDB (overrides config default)
+        save_to_s3: Whether to save to S3 (overrides config default)
+        s3_bucket: S3 bucket name (overrides config default)
+        s3_key_prefix: S3 key prefix (overrides config default)
+    """
+    # Use placeholder names if not displaying file names in logs
+    if DISPLAY_FILE_NAMES_IN_LOGS != 'True':
+        if doc_file_name:
+            doc_file_name = "document"
+            data_file_name = ""
+        if data_file_name:
+            data_file_name = "data_file"
+            doc_file_name = ""
+    else:
+        doc_file_name = doc_file_name
+        data_file_name = data_file_name
+
+    rounded_time_taken = round(time_taken, 2)
+    
+    data = [
+        session_hash,
+        doc_file_name,
+        data_file_name,
+        rounded_time_taken,
+        total_pages,
+        textract_queries,
+        pii_method,
+        comprehend_queries,
+        cost_code,
+        handwriting_signature,
+        HOST_NAME,
+        text_extraction_method,
+        is_textract_call,
+        task
+    ]
+    
+    logger.log_usage(
+        data, 
+        save_to_dynamodb=save_to_dynamodb,
+        save_to_s3=save_to_s3,
+        s3_bucket=s3_bucket,
+        s3_key_prefix=s3_key_prefix
+    )

tools/config.py

@@ -34,7 +34,7 @@ def add_folder_to_path(folder_path: str):
     '''
 
     if os.path.exists(folder_path) and os.path.isdir(folder_path):
-        print(folder_path, "folder exists.")
+        #print(folder_path, "folder exists.")
 
         # Resolve relative path to absolute path
         absolute_path = os.path.abspath(folder_path)
@@ -45,7 +45,8 @@ def add_folder_to_path(folder_path: str):
             os.environ['PATH'] = full_path_extension
             #print(f"Updated PATH with: ", full_path_extension)
         else:
-            print(f"Directory {folder_path} already exists in PATH.")
+            pass
+            #print(f"Directory {folder_path} already exists in PATH.")
     else:
         print(f"Folder not found at {folder_path} - not added to PATH")
 
@@ -88,13 +89,16 @@ AWS_CLIENT_SECRET = get_or_create_env_var('AWS_CLIENT_SECRET', '')
 AWS_USER_POOL_ID = get_or_create_env_var('AWS_USER_POOL_ID', '')
 
 AWS_ACCESS_KEY = get_or_create_env_var('AWS_ACCESS_KEY', '')
-if AWS_ACCESS_KEY: print(f'AWS_ACCESS_KEY found in environment variables')
+#if AWS_ACCESS_KEY: print(f'AWS_ACCESS_KEY found in environment variables')
 
 AWS_SECRET_KEY = get_or_create_env_var('AWS_SECRET_KEY', '')
-if AWS_SECRET_KEY: print(f'AWS_SECRET_KEY found in environment variables')
+#if AWS_SECRET_KEY: print(f'AWS_SECRET_KEY found in environment variables')
 
 DOCUMENT_REDACTION_BUCKET = get_or_create_env_var('DOCUMENT_REDACTION_BUCKET', '')
 
+# Should the app prioritise using AWS SSO over using API keys stored in environment variables/secrets (defaults to yes)
+PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS = get_or_create_env_var('PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS', '1')
+
 # Custom headers e.g. if routing traffic through Cloudfront
 # Retrieving or setting CUSTOM_HEADER
 CUSTOM_HEADER = get_or_create_env_var('CUSTOM_HEADER', '')
@@ -164,7 +168,7 @@ DISPLAY_FILE_NAMES_IN_LOGS = get_or_create_env_var('DISPLAY_FILE_NAMES_IN_LOGS',
 # Further customisation options for CSV logs
 CSV_ACCESS_LOG_HEADERS = get_or_create_env_var('CSV_ACCESS_LOG_HEADERS', '') # If blank, uses component labels
 CSV_FEEDBACK_LOG_HEADERS = get_or_create_env_var('CSV_FEEDBACK_LOG_HEADERS', '') # If blank, uses component labels
-CSV_USAGE_LOG_HEADERS = get_or_create_env_var('CSV_USAGE_LOG_HEADERS', '["session_hash_textbox", "doc_full_file_name_textbox", "data_full_file_name_textbox", "actual_time_taken_number",	"total_page_count",	"textract_query_number", "pii_detection_method", "comprehend_query_number",  "cost_code", "textract_handwriting_signature", "host_name_textbox", "text_extraction_method", "is_this_a_textract_api_call"]') # If blank, uses component labels
+CSV_USAGE_LOG_HEADERS = get_or_create_env_var('CSV_USAGE_LOG_HEADERS', '["session_hash_textbox", "doc_full_file_name_textbox", "data_full_file_name_textbox", "actual_time_taken_number",	"total_page_count",	"textract_query_number", "pii_detection_method", "comprehend_query_number",  "cost_code", "textract_handwriting_signature", "host_name_textbox", "text_extraction_method", "is_this_a_textract_api_call", "task"]') # If blank, uses component labels
 
 ### DYNAMODB logs. Whether to save to DynamoDB, and the headers of the table
 SAVE_LOGS_TO_DYNAMODB = get_or_create_env_var('SAVE_LOGS_TO_DYNAMODB', 'False')
@@ -275,7 +279,7 @@ DEFAULT_TABULAR_ANONYMISATION_STRATEGY = get_or_create_env_var('DEFAULT_TABULAR_
 ### Local OCR model - Tesseract vs PaddleOCR
 CHOSEN_LOCAL_OCR_MODEL = get_or_create_env_var('CHOSEN_LOCAL_OCR_MODEL', "tesseract") # Choose between "tesseract", "hybrid", and "paddle". "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. "hybrid" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence.
 
-PREPROCESS_LOCAL_OCR_IMAGES = get_or_create_env_var('PREPROCESS_LOCAL_OCR_IMAGES', "False") # Whether to try and preprocess images before extracting text. NOTE: I have found in testing that this often results in WORSE results for scanned pages, so it is default False
+PREPROCESS_LOCAL_OCR_IMAGES = get_or_create_env_var('PREPROCESS_LOCAL_OCR_IMAGES', "True") # Whether to try and preprocess images before extracting text. NOTE: I have found in testing that this often results in WORSE results for scanned pages, so it is default False
 
 # Entities for redaction
 CHOSEN_COMPREHEND_ENTITIES = get_or_create_env_var('CHOSEN_COMPREHEND_ENTITIES', "['BANK_ACCOUNT_NUMBER','BANK_ROUTING','CREDIT_DEBIT_NUMBER','CREDIT_DEBIT_CVV','CREDIT_DEBIT_EXPIRY','PIN','EMAIL','ADDRESS','NAME','PHONE', 'PASSPORT_NUMBER','DRIVER_ID', 'USERNAME','PASSWORD', 'IP_ADDRESS','MAC_ADDRESS', 'LICENSE_PLATE','VEHICLE_IDENTIFICATION_NUMBER','UK_NATIONAL_INSURANCE_NUMBER', 'INTERNATIONAL_BANK_ACCOUNT_NUMBER','SWIFT_CODE','UK_NATIONAL_HEALTH_SERVICE_NUMBER']")
@@ -287,6 +291,10 @@ CHOSEN_REDACT_ENTITIES = get_or_create_env_var('CHOSEN_REDACT_ENTITIES', "['TITL
 
 FULL_ENTITY_LIST = get_or_create_env_var('FULL_ENTITY_LIST', "['TITLES', 'PERSON', 'PHONE_NUMBER', 'EMAIL_ADDRESS', 'STREETNAME', 'UKPOSTCODE', 'CREDIT_CARD', 'CRYPTO', 'DATE_TIME', 'IBAN_CODE', 'IP_ADDRESS', 'NRP', 'LOCATION', 'MEDICAL_LICENSE', 'URL', 'UK_NHS', 'CUSTOM', 'CUSTOM_FUZZY']")
 
+CUSTOM_ENTITIES = get_or_create_env_var('CUSTOM_ENTITIES', "['TITLES', 'UKPOSTCODE', 'STREETNAME', 'CUSTOM']")
+
+
+
 DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX = get_or_create_env_var('DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX', "['Extract handwriting']")
 
 DEFAULT_SEARCH_QUERY = get_or_create_env_var('DEFAULT_SEARCH_QUERY', '')
@@ -327,8 +335,9 @@ LANGUAGE_CHOICES = get_or_create_env_var("LANGUAGE_CHOICES", "['en', 'fr', 'de',
 DEFAULT_DUPLICATE_DETECTION_THRESHOLD = float(get_or_create_env_var("DEFAULT_DUPLICATE_DETECTION_THRESHOLD", "0.95"))
 DEFAULT_MIN_CONSECUTIVE_PAGES = int(get_or_create_env_var("DEFAULT_MIN_CONSECUTIVE_PAGES", "1"))
 USE_GREEDY_DUPLICATE_DETECTION = get_or_create_env_var("USE_GREEDY_DUPLICATE_DETECTION", "True")
-DEFAULT_COMBINE_PAGES = get_or_create_env_var("DEFAULT_COMBINE_PAGES", "True")
+DEFAULT_COMBINE_PAGES = get_or_create_env_var("DEFAULT_COMBINE_PAGES", "True") # Combine text from the same page number within a file. Alternative will enable line-level duplicate detection.
 DEFAULT_MIN_WORD_COUNT = int(get_or_create_env_var("DEFAULT_MIN_WORD_COUNT", "10"))
+REMOVE_DUPLICATE_ROWS = get_or_create_env_var("REMOVE_DUPLICATE_ROWS", "False")
 
 
 ###
@@ -352,6 +361,7 @@ COGNITO_AUTH = get_or_create_env_var('COGNITO_AUTH', '0')
 RUN_DIRECT_MODE = get_or_create_env_var('RUN_DIRECT_MODE', '0')
 
 # Direct mode configuration options
+DIRECT_MODE_DEFAULT_USER = get_or_create_env_var('DIRECT_MODE_DEFAULT_USER', '') # Default username for cli/direct mode requests
 DIRECT_MODE_TASK = get_or_create_env_var('DIRECT_MODE_TASK', 'redact')  # 'redact' or 'deduplicate'
 DIRECT_MODE_INPUT_FILE = get_or_create_env_var('DIRECT_MODE_INPUT_FILE', '')  # Path to input file
 DIRECT_MODE_OUTPUT_DIR = get_or_create_env_var('DIRECT_MODE_OUTPUT_DIR', OUTPUT_FOLDER)  # Output directory
@@ -447,4 +457,4 @@ TEXTRACT_JOBS_S3_INPUT_LOC = get_or_create_env_var('TEXTRACT_JOBS_S3_INPUT_LOC',
 
 TEXTRACT_JOBS_LOCAL_LOC = get_or_create_env_var('TEXTRACT_JOBS_LOCAL_LOC', 'output') # Local subfolder where the Textract jobs are stored
 
-DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS = get_or_create_env_var('DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS', '7') # How many days into the past should whole document Textract jobs be displayed? After that, the data is not deleted from the Textract jobs csv, but it is just filtered out. Included to align with S3 buckets where the file outputs will be automatically deleted after X days.
+DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS = int(get_or_create_env_var('DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS', '7')) # How many days into the past should whole document Textract jobs be displayed? After that, the data is not deleted from the Textract jobs csv, but it is just filtered out. Included to align with S3 buckets where the file outputs will be automatically deleted after X days.

tools/custom_csvlogger.py

@@ -15,7 +15,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from gradio_client import utils as client_utils
 import gradio as gr
-from gradio import utils, wasm_utils
+from gradio import utils
 from tools.config import AWS_REGION, AWS_ACCESS_KEY, AWS_SECRET_KEY, RUN_AWS_FUNCTIONS
 
 
@@ -56,9 +56,7 @@ class CSVLogger_custom(FlaggingCallback):
         self.simplify_file_data = simplify_file_data
         self.verbose = verbose
         self.dataset_file_name = dataset_file_name
-        self.lock = (
-            Lock() if not wasm_utils.IS_WASM else contextlib.nullcontext()
-        )  # The multiprocessing module doesn't work on Lite.
+        self.lock = Lock()
 
     def setup(
         self,

tools/custom_image_analyser_engine.py

@@ -260,10 +260,12 @@ class ContrastSegmentedImageEnhancer(ImagePreprocessor):
             adjusted_contrast = contrast
         return adjusted_image, contrast, adjusted_contrast
 
-    def preprocess_image(self, image: Image.Image) -> Tuple[Image.Image, dict]:
+    def preprocess_image(self, image: Image.Image, perform_binarization: bool = False) -> Tuple[Image.Image, dict]:
         """
         A corrected, logical pipeline for OCR preprocessing.
         Order: Greyscale -> Rescale -> Denoise -> Enhance Contrast -> Binarize
+
+        I have found that binarization is not always helpful with Tesseract, and can sometimes degrade results. So it is off by default.
         """
         # 1. Convert to greyscale NumPy array
         image_np = self.convert_image_to_array(image)
@@ -278,9 +280,13 @@ class ContrastSegmentedImageEnhancer(ImagePreprocessor):
         adjusted_image_np, _, _ = self._improve_contrast(filtered_image_np)
 
         # 5. Adaptive Thresholding (Binarization) - This is the final step
-        final_image_np, threshold_metadata = self.adaptive_threshold.preprocess_image(
-            adjusted_image_np
-        )
+        if perform_binarization:
+            final_image_np, threshold_metadata = self.adaptive_threshold.preprocess_image(
+                adjusted_image_np
+            )
+        else:
+            final_image_np = adjusted_image_np
+            threshold_metadata = {}
         
         # Combine metadata
         final_metadata = {**scale_metadata, **threshold_metadata}

tools/data_anonymise.py

@@ -18,15 +18,19 @@ from botocore.client import BaseClient
 from presidio_anonymizer.entities import OperatorConfig, ConflictResolutionStrategy
 from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, DictAnalyzerResult, RecognizerResult
 from presidio_anonymizer import AnonymizerEngine, BatchAnonymizerEngine
-from tools.config import RUN_AWS_FUNCTIONS, AWS_ACCESS_KEY, AWS_SECRET_KEY, OUTPUT_FOLDER, DEFAULT_LANGUAGE, aws_comprehend_language_choices, DO_INITIAL_TABULAR_DATA_CLEAN
-from tools.helper_functions import get_file_name_without_type, read_file, detect_file_type
-from tools.load_spacy_model_custom_recognisers import nlp_analyser, score_threshold, custom_word_list_recogniser, CustomWordFuzzyRecognizer, custom_entities, create_nlp_analyser, load_spacy_model
+from tools.config import RUN_AWS_FUNCTIONS, AWS_ACCESS_KEY, AWS_SECRET_KEY, OUTPUT_FOLDER, DEFAULT_LANGUAGE, aws_comprehend_language_choices, DO_INITIAL_TABULAR_DATA_CLEAN, CUSTOM_ENTITIES, PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS, AWS_REGION
+from tools.helper_functions import get_file_name_without_type, read_file, detect_file_type, _get_env_list
+from tools.load_spacy_model_custom_recognisers import nlp_analyser, score_threshold, custom_word_list_recogniser, CustomWordFuzzyRecognizer,  create_nlp_analyser, load_spacy_model
 # Use custom version of analyze_dict to be able to track progress
 from tools.presidio_analyzer_custom import analyze_dict
 
 if DO_INITIAL_TABULAR_DATA_CLEAN == "True": DO_INITIAL_TABULAR_DATA_CLEAN = True 
 else: DO_INITIAL_TABULAR_DATA_CLEAN = False
 
+if CUSTOM_ENTITIES: CUSTOM_ENTITIES = _get_env_list(CUSTOM_ENTITIES)
+
+custom_entities = CUSTOM_ENTITIES
+
 fake = Faker("en_UK")
 def fake_first_name(x):
     return fake.first_name()
@@ -233,7 +237,7 @@ def anon_consistent_names(df:pd.DataFrame) -> pd.DataFrame:
 def handle_docx_anonymisation(
     file_path: str,
     output_folder: str,
-    anon_strat: str,
+    anon_strategy: str,
     chosen_redact_entities: List[str],
     in_allow_list: List[str],
     in_deny_list: List[str],
@@ -274,15 +278,15 @@ def handle_docx_anonymisation(
     # If there's no text to process, return early
     if not original_texts:
         print(f"No text found in {file_path}. Skipping.")
-        return None, None
+        return None, None, 0
 
     # 2. Convert to a DataFrame for the existing anonymisation script
     df_to_anonymise = pd.DataFrame({'text_to_redact': original_texts})
     
     # 3. Call the core anonymisation script
-    anonymised_df, _, decision_log = anonymise_script(
+    anonymised_df, _, decision_log, comprehend_query_number = anonymise_script(
         df=df_to_anonymise,
-        anon_strat=anon_strat,
+        anon_strategy=anon_strategy,
         language=language,
         chosen_redact_entities=chosen_redact_entities,
         in_allow_list=in_allow_list,
@@ -322,11 +326,11 @@ def handle_docx_anonymisation(
     with open(log_file_path, "w", encoding="utf-8-sig") as f:
         f.write(decision_log)
 
-    return output_docx_path, log_file_path, output_xlsx_path
+    return output_docx_path, log_file_path, output_xlsx_path, comprehend_query_number
 
 def anonymise_files_with_open_text(file_paths: List[str], 
                          in_text: str, 
-                         anon_strat: str, 
+                         anon_strategy: str, 
                          chosen_cols: List[str],
                          chosen_redact_entities: List[str], 
                          in_allow_list: List[str] = None, 
@@ -354,7 +358,7 @@ def anonymise_files_with_open_text(file_paths: List[str],
     Parameters:
     - file_paths (List[str]): A list of file paths to anonymise: '.xlsx', '.xls', '.csv', '.parquet', or '.docx'.
     - in_text (str): The text to anonymise if file_paths is 'open_text'.
-    - anon_strat (str): The anonymisation strategy to use.
+    - anon_strategy (str): The anonymisation strategy to use.
     - chosen_cols (List[str]): A list of column names to anonymise.
     - language (str): The language of the text to anonymise.
     - chosen_redact_entities (List[str]): A list of entities to redact.
@@ -381,6 +385,9 @@ def anonymise_files_with_open_text(file_paths: List[str],
     
     tic = time.perf_counter()
     comprehend_client = ""
+
+    # If output folder doesn't end with a forward slash, add one
+    if not output_folder.endswith('/'): output_folder = output_folder + '/'
     
     # Use provided language or default
     language = language or DEFAULT_LANGUAGE
@@ -427,7 +434,10 @@ def anonymise_files_with_open_text(file_paths: List[str],
      # Try to connect to AWS services directly only if RUN_AWS_FUNCTIONS environmental variable is 1, otherwise an environment variable or direct textbox input is needed.
     if pii_identification_method == "AWS Comprehend":
         print("Trying to connect to AWS Comprehend service")
-        if aws_access_key_textbox and aws_secret_key_textbox:
+        if RUN_AWS_FUNCTIONS == "1" and PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS == "1":
+            print("Connecting to Comprehend via existing SSO connection")
+            comprehend_client = boto3.client('comprehend', region_name=AWS_REGION)
+        elif aws_access_key_textbox and aws_secret_key_textbox:
             print("Connecting to Comprehend using AWS access key and secret keys from textboxes.")
             print("aws_access_key_textbox:", aws_access_key_textbox)
             print("aws_secret_access_key:", aws_secret_key_textbox)
@@ -459,14 +469,18 @@ def anonymise_files_with_open_text(file_paths: List[str],
     if latest_file_completed >= len(file_paths):
         print("Last file reached") #, returning files:", str(latest_file_completed))
         # Set to a very high number so as not to mess with subsequent file processing by the user
-        latest_file_completed = 99
+        #latest_file_completed = 99
         final_out_message = '\n'.join(out_message)
-        return final_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, actual_time_taken_number
+        return final_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, actual_time_taken_number, comprehend_query_number
     
     file_path_loop = [file_paths[int(latest_file_completed)]]
-        
+
     for anon_file in progress.tqdm(file_path_loop, desc="Anonymising files", unit = "files"):
 
+        # Get a string file path
+        if isinstance(anon_file, str): file_path = anon_file
+        else: file_path = anon_file 
+
         if anon_file=='open_text':
             anon_df = pd.DataFrame(data={'text':[in_text]})
             chosen_cols=['text']
@@ -474,19 +488,19 @@ def anonymise_files_with_open_text(file_paths: List[str],
             sheet_name = ""
             file_type = ""
 
-            out_file_paths, out_message, key_string, log_files_output_paths = tabular_anonymise_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strat, language, chosen_redact_entities, in_allow_list, file_type, "", log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=OUTPUT_FOLDER, do_initial_clean=do_initial_clean)
+            out_file_paths, out_message, key_string, log_files_output_paths, comprehend_query_number = tabular_anonymise_wrapper_func(file_path, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strategy, language, chosen_redact_entities, in_allow_list, file_type, "", log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=OUTPUT_FOLDER, do_initial_clean=do_initial_clean)
         else:
             # If file is an xlsx, we are going to run through all the Excel sheets to anonymise them separately.
-            file_type = detect_file_type(anon_file)
+            file_type = detect_file_type(file_path)
             print("File type is:", file_type)
 
-            out_file_part = get_file_name_without_type(anon_file.name)
+            out_file_part = get_file_name_without_type(file_path)
 
             if file_type == 'docx':
-                output_path, log_path, output_xlsx_path = handle_docx_anonymisation(
-                    file_path=anon_file.name, # .name if it's a temp file object
+                output_path, log_path, output_xlsx_path, comprehend_query_number = handle_docx_anonymisation(
+                    file_path=file_path,
                     output_folder=output_folder,
-                    anon_strat=anon_strat,
+                    anon_strategy=anon_strategy,
                     chosen_redact_entities=chosen_redact_entities,
                     in_allow_list=in_allow_list_flat,
                     in_deny_list=in_deny_list,
@@ -512,7 +526,7 @@ def anonymise_files_with_open_text(file_paths: List[str],
                     continue
 
                 # Create xlsx file:
-                anon_xlsx = pd.ExcelFile(anon_file)                
+                anon_xlsx = pd.ExcelFile(file_path)                
                 anon_xlsx_export_file_name = output_folder + out_file_part + "_redacted.xlsx"                
 
                 
@@ -523,16 +537,16 @@ def anonymise_files_with_open_text(file_paths: List[str],
                     if sheet_name not in anon_xlsx.sheet_names:
                         continue
 
-                    anon_df = pd.read_excel(anon_file, sheet_name=sheet_name)
+                    anon_df = pd.read_excel(file_path, sheet_name=sheet_name)
 
-                    out_file_paths, out_message, key_string, log_files_output_paths  = tabular_anonymise_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strat, language, chosen_redact_entities, in_allow_list, file_type, anon_xlsx_export_file_name, log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, language, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=output_folder, do_initial_clean=do_initial_clean)
+                    out_file_paths, out_message, key_string, log_files_output_paths, comprehend_query_number  = tabular_anonymise_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strategy, language, chosen_redact_entities, in_allow_list, file_type, anon_xlsx_export_file_name, log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, language, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=output_folder, do_initial_clean=do_initial_clean)
 
             else:
                 sheet_name = ""
-                anon_df = read_file(anon_file)
-                out_file_part = get_file_name_without_type(anon_file.name)
+                anon_df = read_file(file_path)
+                out_file_part = get_file_name_without_type(file_path)
 
-                out_file_paths, out_message, key_string, log_files_output_paths = tabular_anonymise_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strat, language, chosen_redact_entities, in_allow_list, file_type, "", log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, language, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=output_folder, do_initial_clean=do_initial_clean)
+                out_file_paths, out_message, key_string, log_files_output_paths, comprehend_query_number = tabular_anonymise_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strategy, language, chosen_redact_entities, in_allow_list, file_type, "", log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, language, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=output_folder, do_initial_clean=do_initial_clean)
 
         # Increase latest file completed count unless we are at the last file
         if latest_file_completed != len(file_paths):
@@ -554,14 +568,14 @@ def anonymise_files_with_open_text(file_paths: List[str],
         out_message_out = '\n'.join(out_message)
         out_message_out = out_message_out + " " + out_time
 
-        if anon_strat == "encrypt":
+        if anon_strategy == "encrypt":
             out_message_out.append(". Your decryption key is " + key_string)        
 
         out_message_out = out_message_out + "\n\nGo to to the Redaction settings tab to see redaction logs. Please give feedback on the results below to help improve this app."
 
         out_message_out = re.sub(r'^\n+|^\. ', '', out_message_out).strip()
     
-    return out_message_out, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, actual_time_taken_number
+    return out_message_out, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, actual_time_taken_number, comprehend_query_number
 
 def tabular_anonymise_wrapper_func(
     anon_file: str, 
@@ -571,7 +585,7 @@ def tabular_anonymise_wrapper_func(
     out_file_part: str, 
     out_message: str, 
     excel_sheet_name: str, 
-    anon_strat: str, 
+    anon_strategy: str, 
     language: str,
     chosen_redact_entities: List[str], 
     in_allow_list: List[str], 
@@ -600,7 +614,7 @@ def tabular_anonymise_wrapper_func(
     - out_file_part: A part of the output file name.
     - out_message: A message to be displayed during the anonymization process.
     - excel_sheet_name: The name of the Excel sheet where the anonymized data will be exported.
-    - anon_strat: The anonymization strategy to be applied.
+    - anon_strategy: The anonymization strategy to be applied.
     - language: The language of the data to be anonymized.
     - chosen_redact_entities: A list of entities to be redacted.
     - in_allow_list: A list of allowed values.
@@ -648,7 +662,7 @@ def tabular_anonymise_wrapper_func(
         out_message = "No chosen columns found in dataframe: " + out_file_part
         key_string = ""
         print(out_message)
-        return out_file_paths, out_message, key_string, log_files_output_paths
+        return out_file_paths, out_message, key_string, log_files_output_paths, comprehend_query_number
     else:
         chosen_cols_in_anon_df = get_common_strings(chosen_cols, all_cols_original_order)
 
@@ -663,7 +677,7 @@ def tabular_anonymise_wrapper_func(
 
         
     # Anonymise the selected columns
-    anon_df_part_out, key_string, decision_process_output_str = anonymise_script(anon_df_part, anon_strat, language, chosen_redact_entities, in_allow_list, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, nlp_analyser=nlp_analyser, do_initial_clean=do_initial_clean)
+    anon_df_part_out, key_string, decision_process_output_str, comprehend_query_number = anonymise_script(anon_df_part, anon_strategy, language, chosen_redact_entities, in_allow_list, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, nlp_analyser=nlp_analyser, do_initial_clean=do_initial_clean)
 
     anon_df_part_out.replace("^nan$", "", regex=True, inplace=True)
 
@@ -673,10 +687,10 @@ def tabular_anonymise_wrapper_func(
     
     # Export file
     #  Rename anonymisation strategy for file path naming
-    if anon_strat == "replace with 'REDACTED'": anon_strat_txt = "redact_replace"
-    elif anon_strat == "replace with <ENTITY_NAME>": anon_strat_txt = "redact_entity_type"
-    elif anon_strat == "redact completely": anon_strat_txt = "redact_remove"
-    else: anon_strat_txt = anon_strat
+    if anon_strategy == "replace with 'REDACTED'": anon_strat_txt = "redact_replace"
+    elif anon_strategy == "replace with <ENTITY_NAME>": anon_strat_txt = "redact_entity_type"
+    elif anon_strategy == "redact completely": anon_strat_txt = "redact_remove"
+    else: anon_strat_txt = anon_strategy
 
     # If the file is an xlsx, add a new sheet to the existing xlsx. Otherwise, write to csv
     if file_type == 'xlsx':
@@ -716,10 +730,10 @@ def tabular_anonymise_wrapper_func(
     if anon_file=='open_text':
         out_message = ["'" + anon_df_out['text'][0] + "'"]
 
-    return out_file_paths, out_message, key_string, log_files_output_paths
+    return out_file_paths, out_message, key_string, log_files_output_paths, comprehend_query_number
        
 def anonymise_script(df:pd.DataFrame,
-                     anon_strat:str,
+                     anon_strategy:str,
                      language:str,
                      chosen_redact_entities:List[str],
                      in_allow_list:List[str]=list(),
@@ -738,7 +752,7 @@ def anonymise_script(df:pd.DataFrame,
 
     Args:
         df (pd.DataFrame): The input DataFrame containing text to be anonymised.
-        anon_strat (str): The anonymisation strategy to apply (e.g., "replace with 'REDACTED'", "replace with <ENTITY_NAME>", "redact completely").
+        anon_strategy (str): The anonymisation strategy to apply (e.g., "replace with 'REDACTED'", "replace with <ENTITY_NAME>", "redact completely").
         language (str): The language of the text for analysis (e.g., "en", "es").
         chosen_redact_entities (List[str]): A list of entity types to redact using the local (Presidio) method.
         in_allow_list (List[str], optional): A list of terms to explicitly allow and not redact. Defaults to an empty list.
@@ -948,12 +962,15 @@ def anonymise_script(df:pd.DataFrame,
     people_encrypt_config = eval('{"PERSON": OperatorConfig("encrypt", {"key": key_string})}') # The encryption is using AES cypher in CBC mode and requires a cryptographic key as an input for both the encryption and the decryption.
     fake_first_name_config = eval('{"PERSON": OperatorConfig("custom", {"lambda": fake_first_name})}')
 
-    if anon_strat == "replace with 'REDACTED'": chosen_mask_config = simple_replace_config
-    if anon_strat == "replace with <ENTITY_NAME>": chosen_mask_config = replace_config
-    if anon_strat == "redact completely": chosen_mask_config = redact_config
-    if anon_strat == "hash": chosen_mask_config = hash_config
-    if anon_strat == "mask": chosen_mask_config = mask_config
-    if anon_strat == "encrypt": 
+    if anon_strategy == "replace with 'REDACTED'": chosen_mask_config = simple_replace_config
+    elif anon_strategy == "replace_redacted": chosen_mask_config = simple_replace_config   
+    elif anon_strategy == "replace with <ENTITY_NAME>": chosen_mask_config = replace_config
+    elif anon_strategy == "entity_type": chosen_mask_config = replace_config
+    elif anon_strategy == "redact completely": chosen_mask_config = redact_config
+    elif anon_strategy == "redact": chosen_mask_config = redact_config
+    elif anon_strategy == "hash": chosen_mask_config = hash_config
+    elif anon_strategy == "mask": chosen_mask_config = mask_config
+    elif anon_strategy == "encrypt": 
         chosen_mask_config = people_encrypt_config
         key = secrets.token_bytes(16)  # 128 bits = 16 bytes
         key_string = base64.b64encode(key).decode('utf-8')
@@ -962,7 +979,10 @@ def anonymise_script(df:pd.DataFrame,
         for entity, operator in chosen_mask_config.items():
             if operator.operator_name == "encrypt":
                 operator.params = {"key": key_string}
-    elif anon_strat == "fake_first_name": chosen_mask_config = fake_first_name_config
+    elif anon_strategy == "fake_first_name": chosen_mask_config = fake_first_name_config
+    else:
+        print("Anonymisation strategy not found. Redacting completely by default.")
+        chosen_mask_config = redact_config # Redact completely by default
 
     # I think in general people will want to keep date / times - removed Mar 2025 as I don't want to assume for people.
     #keep_date_config = eval('{"DATE_TIME": OperatorConfig("keep")}')
@@ -973,4 +993,4 @@ def anonymise_script(df:pd.DataFrame,
 
     scrubbed_df = pd.DataFrame(anonymizer_results)
     
-    return scrubbed_df, key_string, decision_process_output_str
+    return scrubbed_df, key_string, decision_process_output_str, comprehend_query_number

tools/example_cli_calls.txt

@@ -1,30 +0,0 @@
-python cli_redact.py --help
-
-python cli_redact.py \
-    --input_file "documents/confidential-report.pdf" \
-    --output_dir "output/redacted_reports/" \
-    --ocr_method "Local OCR model - PDFs without selectable text" \
-    --pii_detector "Local" \
-    --page_min 2 \
-    --page_max 10 \
-    --allow_list "config/project_allowlist.csv"
-
-python your_cli_script.py \
-    --input_file "data/customer_data.xlsx" \
-    --output_dir "output/anonymised_data/" \
-    --anon_strat "redact" \
-    --columns "Customer Name" "Email" \
-    --excel_sheets "Q3-Data"
-
-python your_cli_script.py \
-    --input_file "legal_docs/legal_agreement.docx" \
-    --output_dir "output/anonymised_docs/" \
-    --anon_strat "encrypt" \
-    --deny_list "config/codenames.csv" \
-    --language "en"
-
-python cli_redact.py --task deduplicate --input_file ocr_output.csv --duplicate_type pages --similarity_threshold 0.95 --min_word_count 5
-
-python cli_redact.py --task deduplicate --input_file data.csv --duplicate_type tabular --text_columns "Name" "Email" "Description"
-
-python cli_redact.py --task deduplicate --input_file ocr_output.csv --duplicate_type pages --search_query "confidential information"

tools/file_redaction.py

@@ -20,11 +20,11 @@ import gradio as gr
 from gradio import Progress
 from collections import defaultdict  # For efficient grouping
 
-from tools.config import OUTPUT_FOLDER, IMAGES_DPI, MAX_IMAGE_PIXELS, RUN_AWS_FUNCTIONS, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, PAGE_BREAK_VALUE, MAX_TIME_VALUE, LOAD_TRUNCATED_IMAGES, INPUT_FOLDER, RETURN_PDF_END_OF_REDACTION, TESSERACT_TEXT_EXTRACT_OPTION, SELECTABLE_TEXT_EXTRACT_OPTION, TEXTRACT_TEXT_EXTRACT_OPTION, LOCAL_PII_OPTION, AWS_PII_OPTION, NO_REDACTION_PII_OPTION, DEFAULT_LANGUAGE, textract_language_choices, aws_comprehend_language_choices
+from tools.config import OUTPUT_FOLDER, IMAGES_DPI, MAX_IMAGE_PIXELS, RUN_AWS_FUNCTIONS, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, PAGE_BREAK_VALUE, MAX_TIME_VALUE, LOAD_TRUNCATED_IMAGES, INPUT_FOLDER, RETURN_PDF_END_OF_REDACTION, TESSERACT_TEXT_EXTRACT_OPTION, SELECTABLE_TEXT_EXTRACT_OPTION, TEXTRACT_TEXT_EXTRACT_OPTION, LOCAL_PII_OPTION, AWS_PII_OPTION, NO_REDACTION_PII_OPTION, DEFAULT_LANGUAGE, textract_language_choices, aws_comprehend_language_choices, CUSTOM_ENTITIES, PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS
 from tools.custom_image_analyser_engine import CustomImageAnalyzerEngine, OCRResult, combine_ocr_results, CustomImageRecognizerResult, run_page_text_redaction,  recreate_page_line_level_ocr_results_with_page
 from tools.file_conversion import convert_annotation_json_to_review_df, redact_whole_pymupdf_page, redact_single_box, is_pdf, is_pdf_or_image, prepare_image_or_pdf, divide_coordinates_by_page_sizes, convert_annotation_data_to_dataframe, divide_coordinates_by_page_sizes, create_annotation_dicts_from_annotation_df, remove_duplicate_images_with_blank_boxes, fill_missing_ids, fill_missing_box_ids, load_and_convert_ocr_results_with_words_json, save_pdf_with_or_without_compression, word_level_ocr_output_to_dataframe
-from tools.load_spacy_model_custom_recognisers import nlp_analyser, score_threshold, custom_entities, custom_recogniser, custom_word_list_recogniser, CustomWordFuzzyRecognizer, load_spacy_model, download_tesseract_lang_pack, create_nlp_analyser
-from tools.helper_functions import get_file_name_without_type, clean_unicode_text
+from tools.load_spacy_model_custom_recognisers import nlp_analyser, score_threshold, custom_recogniser, custom_word_list_recogniser, CustomWordFuzzyRecognizer, load_spacy_model, download_tesseract_lang_pack, create_nlp_analyser
+from tools.helper_functions import get_file_name_without_type, clean_unicode_text, _get_env_list
 from tools.aws_textract import analyse_page_with_textract, json_to_ocrresult, load_and_convert_textract_json
 
 ImageFile.LOAD_TRUNCATED_IMAGES = LOAD_TRUNCATED_IMAGES.lower() == "true"
@@ -34,6 +34,10 @@ image_dpi = float(IMAGES_DPI)
 
 RETURN_PDF_END_OF_REDACTION = RETURN_PDF_END_OF_REDACTION.lower() == "true"
 
+if CUSTOM_ENTITIES: CUSTOM_ENTITIES = _get_env_list(CUSTOM_ENTITIES)
+
+custom_entities = CUSTOM_ENTITIES
+
 def bounding_boxes_overlap(box1, box2):
     """Check if two bounding boxes overlap."""
     return (box1[0] < box2[2] and box2[0] < box1[2] and
@@ -91,7 +95,7 @@ def choose_and_run_redactor(file_paths:List[str],
  chosen_redact_entities:List[str],
  chosen_redact_comprehend_entities:List[str],
  text_extraction_method:str,
- in_allow_list:List[List[str]]=list(),
+ in_allow_list:List[str]=list(),
  in_deny_list:List[str]=list(), 
  redact_whole_page_list:List[str]=list(),
  latest_file_completed:int=0,
@@ -146,9 +150,9 @@ def choose_and_run_redactor(file_paths:List[str],
     - chosen_redact_entities (List[str]): A list of entity types to redact from the files using the local model (spacy) with Microsoft Presidio.
     - chosen_redact_comprehend_entities (List[str]): A list of entity types to redact from files, chosen from the official list from AWS Comprehend service.
     - text_extraction_method (str): The method to use to extract text from documents.
-    - in_allow_list (List[List[str]], optional): A list of allowed terms for redaction. Defaults to None.
-    - in_deny_list (List[List[str]], optional): A list of denied terms for redaction. Defaults to None.
-    - redact_whole_page_list (List[List[str]], optional): A list of whole page numbers for redaction. Defaults to None.
+    - in_allow_list (List[List[str]], optional): A list of allowed terms for redaction. Defaults to empty list. Can also be entered as a string path to a CSV file, or as a single column pandas dataframe.
+    - in_deny_list (List[List[str]], optional): A list of denied terms for redaction. Defaults to empty list. Can also be entered as a string path to a CSV file, or as a single column pandas dataframe.
+    - redact_whole_page_list (List[List[str]], optional): A list of whole page numbers for redaction. Defaults to empty list. Can also be entered as a string path to a CSV file, or as a single column pandas dataframe.
     - latest_file_completed (int, optional): The index of the last completed file. Defaults to 0.
     - combined_out_message (list, optional): A list to store output messages. Defaults to an empty list.
     - out_file_paths (list, optional): A list to store paths to the output files. Defaults to an empty list.
@@ -202,8 +206,19 @@ def choose_and_run_redactor(file_paths:List[str],
     pdf_file_name_without_ext = ""
     page_break_return = False  
     blank_request_metadata = list()
+    custom_recogniser_word_list_flat = list()
     all_textract_request_metadata = all_request_metadata_str.split('\n') if all_request_metadata_str else []
     review_out_file_paths = [prepared_pdf_file_paths[0]]
+    task_textbox = "redact"
+
+    # CLI mode may provide options to enter method names in a different format
+    if text_extraction_method == "AWS Textract": text_extraction_method = TEXTRACT_TEXT_EXTRACT_OPTION
+    if text_extraction_method == "Local OCR": text_extraction_method = TESSERACT_TEXT_EXTRACT_OPTION
+    if text_extraction_method == "Local text": text_extraction_method = SELECTABLE_TEXT_EXTRACT_OPTION
+    if pii_identification_method == "None": pii_identification_method = NO_REDACTION_PII_OPTION
+
+    # If output folder doesn't end with a forward slash, add one
+    if not output_folder.endswith('/'): output_folder = output_folder + '/'
     
     # Use provided language or default
     language = language or DEFAULT_LANGUAGE
@@ -284,6 +299,8 @@ def choose_and_run_redactor(file_paths:List[str],
         progress(0.95, "Completed last file, performing final checks")
         current_loop_page = 0
 
+        if isinstance(combined_out_message, list): combined_out_message = '\n'.join(combined_out_message)
+
         if isinstance(out_message, list) and out_message:
             combined_out_message = combined_out_message + '\n'.join(out_message)
         elif out_message:
@@ -312,7 +329,7 @@ def choose_and_run_redactor(file_paths:List[str],
 
         page_break_return = True
 
-        return combined_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, estimated_time_taken_state, all_request_metadata_str, pymupdf_doc, annotations_all_pages, current_loop_page, page_break_return, all_page_line_level_ocr_results_df, all_pages_decision_process_table, comprehend_query_number, review_out_file_paths, annotate_max_pages, annotate_max_pages, prepared_pdf_file_paths, pdf_image_file_paths, review_file_state, page_sizes, duplication_file_path_outputs, duplication_file_path_outputs, review_file_path, total_textract_query_number, ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words,  all_page_line_level_ocr_results_with_words_df, review_file_state
+        return combined_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, estimated_time_taken_state, all_request_metadata_str, pymupdf_doc, annotations_all_pages, current_loop_page, page_break_return, all_page_line_level_ocr_results_df, all_pages_decision_process_table, comprehend_query_number, review_out_file_paths, annotate_max_pages, annotate_max_pages, prepared_pdf_file_paths, pdf_image_file_paths, review_file_state, page_sizes, duplication_file_path_outputs, duplication_file_path_outputs, review_file_path, total_textract_query_number, ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words,  all_page_line_level_ocr_results_with_words_df, review_file_state, task_textbox
 
     #if first_loop_state == False:
     # Prepare documents and images as required if they don't already exist
@@ -346,10 +363,11 @@ def choose_and_run_redactor(file_paths:List[str],
             file_paths_loop, text_extraction_method, all_page_line_level_ocr_results_df, all_page_line_level_ocr_results_with_words_df, 0, out_message, True, 
             annotate_max_pages, annotations_all_pages, document_cropboxes, redact_whole_page_list, 
             output_folder=output_folder, prepare_images=prepare_images_flag, page_sizes=page_sizes, pymupdf_doc=pymupdf_doc, input_folder=input_folder 
-        )   
-    
-    page_sizes_df = pd.DataFrame(page_sizes)
+        )
+
 
+    page_sizes_df = pd.DataFrame(page_sizes)
+    
     if page_sizes_df.empty:
         page_sizes_df=pd.DataFrame(columns=["page", "image_path", "image_width", "image_height", "mediabox_width", "mediabox_height", "cropbox_width", "cropbox_height", "original_cropbox"])
     page_sizes_df[["page"]] = page_sizes_df[["page"]].apply(pd.to_numeric, errors="coerce")
@@ -378,32 +396,45 @@ def choose_and_run_redactor(file_paths:List[str],
 
         page_break_return = False
 
-        return combined_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, estimated_time_taken_state, all_request_metadata_str, pymupdf_doc, annotations_all_pages, current_loop_page, page_break_return, all_page_line_level_ocr_results_df, all_pages_decision_process_table, comprehend_query_number, review_out_file_paths, annotate_max_pages, annotate_max_pages, prepared_pdf_file_paths, pdf_image_file_paths, review_file_state, page_sizes, duplication_file_path_outputs, duplication_file_path_outputs, review_file_path, total_textract_query_number, ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df, review_file_state
+        return combined_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, estimated_time_taken_state, all_request_metadata_str, pymupdf_doc, annotations_all_pages, current_loop_page, page_break_return, all_page_line_level_ocr_results_df, all_pages_decision_process_table, comprehend_query_number, review_out_file_paths, annotate_max_pages, annotate_max_pages, prepared_pdf_file_paths, pdf_image_file_paths, review_file_state, page_sizes, duplication_file_path_outputs, duplication_file_path_outputs, review_file_path, total_textract_query_number, ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df, review_file_state, task_textbox
 
-    # Load/create allow list
+    ### Load/create allow list, deny list, and whole page redaction list
+
+    ### Load/create allow list
     # If string, assume file path
-    if isinstance(in_allow_list, str): in_allow_list = pd.read_csv(in_allow_list)
+    if isinstance(in_allow_list, str): 
+        if in_allow_list:
+            in_allow_list = pd.read_csv(in_allow_list, header=None)
     # Now, should be a pandas dataframe format
-    if not in_allow_list.empty:
-        in_allow_list_flat = in_allow_list.iloc[:,0].tolist()
+    if isinstance(in_allow_list, pd.DataFrame):
+        if not in_allow_list.empty:
+            in_allow_list_flat = in_allow_list.iloc[:,0].tolist()
+        else:
+            in_allow_list_flat = list()
     else:
         in_allow_list_flat = list()
 
+    ### Load/create deny list
     # If string, assume file path
     if isinstance(in_deny_list, str):
-        in_deny_list = pd.read_csv(in_deny_list)
+        if in_deny_list:
+            in_deny_list = pd.read_csv(in_deny_list, header=None)
+
     if isinstance(in_deny_list, pd.DataFrame):
         if not in_deny_list.empty:
             custom_recogniser_word_list_flat = in_deny_list.iloc[:, 0].tolist()
         else:
             custom_recogniser_word_list_flat = list()
-
         # Sort the strings in order from the longest string to the shortest
         custom_recogniser_word_list_flat = sorted(custom_recogniser_word_list_flat, key=len, reverse=True)
+    else:
+        custom_recogniser_word_list_flat = list()
 
+    ### Load/create whole page redaction list
     # If string, assume file path
     if isinstance(redact_whole_page_list, str):
-        redact_whole_page_list = pd.read_csv(redact_whole_page_list)
+        if redact_whole_page_list:
+            redact_whole_page_list = pd.read_csv(redact_whole_page_list, header=None)
     if isinstance(redact_whole_page_list, pd.DataFrame):
         if not redact_whole_page_list.empty:
             try:
@@ -412,13 +443,18 @@ def choose_and_run_redactor(file_paths:List[str],
                 print("Could not convert whole page redaction data to number list due to:", e)
                 redact_whole_page_list_flat = redact_whole_page_list.iloc[:,0].tolist()
         else:
-            redact_whole_page_list_flat = list()  
+            redact_whole_page_list_flat = list() 
+    else:
+        redact_whole_page_list_flat = list()
 
-    
+    ### Load/create PII identification method
 
     # Try to connect to AWS services directly only if RUN_AWS_FUNCTIONS environmental variable is 1, otherwise an environment variable or direct textbox input is needed.
     if pii_identification_method == AWS_PII_OPTION:
-        if aws_access_key_textbox and aws_secret_key_textbox:
+        if RUN_AWS_FUNCTIONS == "1" and PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS == "1":
+            print("Connecting to Comprehend via existing SSO connection")
+            comprehend_client = boto3.client('comprehend', region_name=AWS_REGION)
+        elif aws_access_key_textbox and aws_secret_key_textbox:
             print("Connecting to Comprehend using AWS access key and secret keys from user input.")
             comprehend_client = boto3.client('comprehend', 
                 aws_access_key_id=aws_access_key_textbox, 
@@ -441,7 +477,10 @@ def choose_and_run_redactor(file_paths:List[str],
         
     # Try to connect to AWS Textract Client if using that text extraction method
     if text_extraction_method == TEXTRACT_TEXT_EXTRACT_OPTION:   
-        if aws_access_key_textbox and aws_secret_key_textbox:
+        if RUN_AWS_FUNCTIONS == "1" and PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS == "1":
+            print("Connecting to Textract via existing SSO connection")
+            textract_client = boto3.client('textract', region_name=AWS_REGION)
+        elif aws_access_key_textbox and aws_secret_key_textbox:
             print("Connecting to Textract using AWS access key and secret keys from user input.")
             textract_client = boto3.client('textract', 
                 aws_access_key_id=aws_access_key_textbox, 
@@ -647,7 +686,10 @@ def choose_and_run_redactor(file_paths:List[str],
                         print("Saving redacted PDF file:", out_redacted_pdf_file_path)
                         save_pdf_with_or_without_compression(pymupdf_doc, out_redacted_pdf_file_path)
 
-                    out_file_paths.append(out_redacted_pdf_file_path)
+                    if isinstance(out_redacted_pdf_file_path, str):
+                        out_file_paths.append(out_redacted_pdf_file_path)
+                    else:
+                        out_file_paths.append(out_redacted_pdf_file_path[0])
 
             if not all_page_line_level_ocr_results_df.empty:
                 all_page_line_level_ocr_results_df = all_page_line_level_ocr_results_df[["page", "text", "left", "top", "width", "height", "line"]]
@@ -658,8 +700,12 @@ def choose_and_run_redactor(file_paths:List[str],
             all_page_line_level_ocr_results_df.sort_values(["page", "line"], inplace=True)
             all_page_line_level_ocr_results_df.to_csv(ocr_file_path, index = None, encoding="utf-8-sig")
 
-            out_file_paths.append(ocr_file_path)
-            duplication_file_path_outputs.append(ocr_file_path)
+            
+
+            if isinstance(ocr_file_path, str):
+                out_file_paths.append(ocr_file_path)
+            else:
+                duplication_file_path_outputs.append(ocr_file_path[0])
 
             if all_page_line_level_ocr_results_with_words:
                 all_page_line_level_ocr_results_with_words = merge_page_results(all_page_line_level_ocr_results_with_words)
@@ -670,14 +716,12 @@ def choose_and_run_redactor(file_paths:List[str],
                 all_page_line_level_ocr_results_with_words_df = word_level_ocr_output_to_dataframe(all_page_line_level_ocr_results_with_words)
 
                 all_page_line_level_ocr_results_with_words_df = divide_coordinates_by_page_sizes(all_page_line_level_ocr_results_with_words_df, page_sizes_df, xmin="word_x0", xmax="word_x1", ymin="word_y0", ymax="word_y1")
-                # all_page_line_level_ocr_results_with_words_df = divide_coordinates_by_page_sizes(all_page_line_level_ocr_results_with_words_df, page_sizes_df, xmin="line_x0", xmax="line_x1", ymin="line_y0", ymax="line_y1")               
+
+                             
 
                 if text_extraction_method == SELECTABLE_TEXT_EXTRACT_OPTION:
                     # Coordinates need to be reversed for ymin and ymax to match with image annotator objects downstream
                     if not all_page_line_level_ocr_results_with_words_df.empty:
-
-                        # all_page_line_level_ocr_results_with_words_df['line_y0'] = reverse_y_coords(all_page_line_level_ocr_results_with_words_df, 'line_y0')
-                        # all_page_line_level_ocr_results_with_words_df['line_y1'] = reverse_y_coords(all_page_line_level_ocr_results_with_words_df, 'line_y1')
                         all_page_line_level_ocr_results_with_words_df['word_y0'] = reverse_y_coords(all_page_line_level_ocr_results_with_words_df, 'word_y0')
                         all_page_line_level_ocr_results_with_words_df['word_y1'] = reverse_y_coords(all_page_line_level_ocr_results_with_words_df, 'word_y1')
 
@@ -685,20 +729,32 @@ def choose_and_run_redactor(file_paths:List[str],
                 all_page_line_level_ocr_results_with_words_df['line_x0'] = ""
                 all_page_line_level_ocr_results_with_words_df['line_x1'] = ""
                 all_page_line_level_ocr_results_with_words_df['line_y0'] = ""
-                all_page_line_level_ocr_results_with_words_df['line_y1'] = ""
+                all_page_line_level_ocr_results_with_words_df['line_y1'] = ""                
 
                 all_page_line_level_ocr_results_with_words_df.sort_values(["page", "line", "word_x0"], inplace=True)
                 all_page_line_level_ocr_results_with_words_df_file_path = all_page_line_level_ocr_results_with_words_json_file_path.replace(".json", ".csv")
                 all_page_line_level_ocr_results_with_words_df.to_csv(all_page_line_level_ocr_results_with_words_df_file_path, index = None, encoding="utf-8-sig")
 
                 if all_page_line_level_ocr_results_with_words_json_file_path not in log_files_output_paths:
+                    if isinstance(all_page_line_level_ocr_results_with_words_json_file_path, str):
+                        log_files_output_paths.append(all_page_line_level_ocr_results_with_words_json_file_path)
+                    else:
+                        log_files_output_paths.append(all_page_line_level_ocr_results_with_words_json_file_path[0])
                     log_files_output_paths.append(all_page_line_level_ocr_results_with_words_json_file_path)
 
                 if all_page_line_level_ocr_results_with_words_df_file_path not in log_files_output_paths:
-                    log_files_output_paths.append(all_page_line_level_ocr_results_with_words_df_file_path)
+                    if isinstance(all_page_line_level_ocr_results_with_words_df_file_path, str):
+                        log_files_output_paths.append(all_page_line_level_ocr_results_with_words_df_file_path)
+                    else:
+                        log_files_output_paths.append(all_page_line_level_ocr_results_with_words_df_file_path[0])
 
                 if all_page_line_level_ocr_results_with_words_df_file_path not in out_file_paths:
-                    out_file_paths.append(all_page_line_level_ocr_results_with_words_df_file_path)
+                    if isinstance(all_page_line_level_ocr_results_with_words_df_file_path, str):
+                        out_file_paths.append(all_page_line_level_ocr_results_with_words_df_file_path)
+                    else:
+                        out_file_paths.append(all_page_line_level_ocr_results_with_words_df_file_path[0])
+
+                
 
             # Convert the gradio annotation boxes to relative coordinates
             progress(0.93, "Creating review file output")
@@ -711,18 +767,27 @@ def choose_and_run_redactor(file_paths:List[str],
             # Save the gradio_annotation_boxes to a review csv file        
             review_file_state = convert_annotation_json_to_review_df(annotations_all_pages_divide, all_pages_decision_process_table, page_sizes=page_sizes)
 
+
             # Don't need page sizes in outputs
             review_file_state.drop(["image_width", "image_height", "mediabox_width", "mediabox_height", "cropbox_width", "cropbox_height"], axis=1, inplace=True, errors="ignore")
                         
-            review_file_state.to_csv(review_file_path, index=None, encoding="utf-8-sig")
+            if isinstance(review_file_path, str):
+                review_file_state.to_csv(review_file_path, index=None, encoding="utf-8-sig")
+            else:
+                review_file_state.to_csv(review_file_path[0], index=None, encoding="utf-8-sig")
             
-            if pii_identification_method != NO_REDACTION_PII_OPTION: out_file_paths.append(review_file_path)
+            if pii_identification_method != NO_REDACTION_PII_OPTION:
+                if isinstance(review_file_path, str):
+                    out_file_paths.append(review_file_path)
+                else:
+                    out_file_paths.append(review_file_path[0])
+
+            # Make a combined message for the file
+            if isinstance(combined_out_message, list): combined_out_message = '\n'.join(combined_out_message)
+            elif combined_out_message == None: combined_out_message = ""
 
-            # Make a combined message for the file                
-            if isinstance(out_message, list) and out_message:
-                combined_out_message = combined_out_message + '\n'.join(out_message)  # Ensure out_message is a list of strings
-            elif out_message:
-                combined_out_message = combined_out_message + '\n' + out_message
+            if isinstance(out_message, list) and out_message: combined_out_message = combined_out_message + '\n'.join(out_message)
+            elif isinstance(out_message, str) and  out_message: combined_out_message = combined_out_message + '\n' + out_message
 
             toc = time.perf_counter()
             time_taken = toc - tic
@@ -747,7 +812,11 @@ def choose_and_run_redactor(file_paths:List[str],
         with open(all_textract_request_metadata_file_path, "w") as f: f.write(all_request_metadata_str)
 
         # Add the request metadata to the log outputs if not there already
-        if all_textract_request_metadata_file_path not in log_files_output_paths: log_files_output_paths.append(all_textract_request_metadata_file_path)
+        if all_textract_request_metadata_file_path not in log_files_output_paths: 
+            if isinstance(all_textract_request_metadata_file_path, str):
+                log_files_output_paths.append(all_textract_request_metadata_file_path)
+            else:
+                log_files_output_paths.append(all_textract_request_metadata_file_path[0])
 
         new_textract_query_numbers = len(all_textract_request_metadata)
         total_textract_query_number += new_textract_query_numbers
@@ -764,7 +833,7 @@ def choose_and_run_redactor(file_paths:List[str],
 
     page_break_return = True
 
-    return combined_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, estimated_time_taken_state, all_request_metadata_str, pymupdf_doc, annotations_all_pages_divide, current_loop_page, page_break_return, all_page_line_level_ocr_results_df, all_pages_decision_process_table, comprehend_query_number, review_out_file_paths, annotate_max_pages, annotate_max_pages, prepared_pdf_file_paths, pdf_image_file_paths, review_file_state, page_sizes, duplication_file_path_outputs, duplication_file_path_outputs, review_file_path, total_textract_query_number, ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df, review_file_state
+    return combined_out_message, out_file_paths, out_file_paths, latest_file_completed, log_files_output_paths, log_files_output_paths, estimated_time_taken_state, all_request_metadata_str, pymupdf_doc, annotations_all_pages_divide, current_loop_page, page_break_return, all_page_line_level_ocr_results_df, all_pages_decision_process_table, comprehend_query_number, review_out_file_paths, annotate_max_pages, annotate_max_pages, prepared_pdf_file_paths, pdf_image_file_paths, review_file_state, page_sizes, duplication_file_path_outputs, duplication_file_path_outputs, review_file_path, total_textract_query_number, ocr_file_path, all_page_line_level_ocr_results, all_page_line_level_ocr_results_with_words, all_page_line_level_ocr_results_with_words_df, review_file_state, task_textbox
 
 def convert_pikepdf_coords_to_pymupdf(pymupdf_page:Page, pikepdf_bbox, type="pikepdf_annot"):
     '''
@@ -1266,10 +1335,13 @@ def merge_img_bboxes(bboxes: list, combined_results: Dict, page_signature_recogn
 
     # Process signature and handwriting results
     if page_signature_recogniser_results or page_handwriting_recogniser_results:
+
         if "Extract handwriting" in handwrite_signature_checkbox:
+            print("Extracting handwriting in merge_img_bboxes function")
             merged_bboxes.extend(copy.deepcopy(page_handwriting_recogniser_results))
 
         if "Extract signatures" in handwrite_signature_checkbox:
+            print("Extracting signatures in merge_img_bboxes function")
             merged_bboxes.extend(copy.deepcopy(page_signature_recogniser_results))
 
     # Reconstruct bounding boxes for substrings of interest
@@ -2385,7 +2457,10 @@ def redact_text_pdf(
                     all_page_line_text_extraction_characters.extend(line_characters)
                     all_page_line_level_ocr_results_with_words.append(line_level_ocr_results_with_words) 
 
-                page_text_ocr_outputs = pd.concat(page_text_ocr_outputs_list)
+                if page_text_ocr_outputs_list:
+                    page_text_ocr_outputs = pd.concat(page_text_ocr_outputs_list)
+                else:
+                    page_text_ocr_outputs = pd.DataFrame(columns=["page", "text", "left", "top", "width", "height", "line"])
 
                 ### REDACTION
                 if pii_identification_method != NO_REDACTION_PII_OPTION:

tools/find_duplicate_pages.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import os
 import re
-
+import time
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 from typing import List, Tuple, Optional, Dict, Union
@@ -725,7 +725,7 @@ def identify_similar_text_sequences(
     do_text_clean:bool = True,
     file1_name: str = '',
     file2_name: str = '',
-    output_folder: str = "output/",
+    output_folder: str = OUTPUT_FOLDER,
     progress=Progress(track_tqdm=True)
 ) -> Tuple[pd.DataFrame, List[str], pd.DataFrame]:
     """
@@ -870,7 +870,7 @@ def handle_selection_and_preview(evt: gr.SelectData, results_df:pd.DataFrame, fu
     # 3. Return all three outputs in the correct order
     return selected_index, page1_data, page2_data
 
-def exclude_match(results_df:pd.DataFrame, selected_index:int, output_folder="./output/"):
+def exclude_match(results_df:pd.DataFrame, selected_index:int, output_folder=OUTPUT_FOLDER):
     """
     Removes a selected row from the results DataFrame, regenerates output files,
     and clears the text preview panes.
@@ -895,12 +895,16 @@ def exclude_match(results_df:pd.DataFrame, selected_index:int, output_folder="./
     # Return the updated dataframe, the new file list, and clear the preview panes
     return updated_df, new_output_paths, None, None
 
-def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:int, min_consecutive:int, greedy_match:bool, combine_pages:bool=True, preview_length:int=500, progress=gr.Progress(track_tqdm=True)):
+def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:int, min_consecutive:int, greedy_match:bool, combine_pages:bool=True, preview_length:int=500, output_folder:str=OUTPUT_FOLDER, progress=gr.Progress(track_tqdm=True)):
     """
     Wrapper function updated to include the 'greedy_match' boolean.
     """
     if not files:
         raise Warning("Please upload files to analyse.")
+
+    start_time = time.time()
+
+    task_textbox = "deduplicate"
         
     progress(0, desc="Combining input files...")
     df_combined, _, full_out_ocr_df = combine_ocr_output_text(files, combine_pages=combine_pages)
@@ -916,6 +920,7 @@ def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:
         min_consecutive_pages=int(min_consecutive),
         greedy_match=greedy_match,
         combine_pages=combine_pages,
+        output_folder=output_folder,
         progress=progress
     )
 
@@ -929,8 +934,11 @@ def run_duplicate_analysis(files:list[pd.DataFrame], threshold:float, min_words:
 
     if results_df.empty:
         gr.Info(f"No duplicate pages found, no results returned.")
+
+    end_time = time.time()
+    processing_time = round(end_time - start_time, 2)
     
-    return results_df, output_paths, full_data_by_file
+    return results_df, output_paths, full_data_by_file, processing_time, task_textbox
 
 def show_page_previews(full_data_by_file: dict, results_df: pd.DataFrame, evt: gr.SelectData, preview_length:int=500):
     """

tools/find_duplicate_tabular.py

@@ -1,6 +1,7 @@
 import pandas as pd
 import os
 import re
+import time
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 from typing import List, Tuple, Dict
@@ -10,9 +11,10 @@ from pathlib import Path
 from tools.helper_functions import OUTPUT_FOLDER, read_file
 from tools.data_anonymise import initial_clean
 from tools.load_spacy_model_custom_recognisers import nlp
-from tools.config import DO_INITIAL_TABULAR_DATA_CLEAN
+from tools.config import DO_INITIAL_TABULAR_DATA_CLEAN, REMOVE_DUPLICATE_ROWS
 
-similarity_threshold = 0.95
+if REMOVE_DUPLICATE_ROWS == "True": REMOVE_DUPLICATE_ROWS = True
+else: REMOVE_DUPLICATE_ROWS = False
 
 def clean_and_stem_text_series(df: pd.DataFrame, column: str, do_initial_clean_dup: bool = DO_INITIAL_TABULAR_DATA_CLEAN):
     """
@@ -51,13 +53,17 @@ def convert_tabular_data_to_analysis_format(
     Returns:
         List[Tuple[str, pd.DataFrame]]: List containing (file_name, processed_df) tuple
     """
-    if text_columns is None:
-        # Auto-detect text columns (string type columns)
-        text_columns = df.select_dtypes(include=['object', 'string']).columns.tolist()
+    # if text_columns is None:
+    #     # Auto-detect text columns (string type columns)
+    #     print(f"No text columns given for {file_name}")
+    #     return []
+    #     text_columns = df.select_dtypes(include=['object', 'string']).columns.tolist()
+
+    text_columns = [col for col in text_columns if col in df.columns]
     
     if not text_columns:
         print(f"No text columns found in {file_name}")
-        return []
+        return list()
     
     # Create a copy to avoid modifying original
     df_copy = df.copy()
@@ -69,9 +75,9 @@ def convert_tabular_data_to_analysis_format(
     df_copy['row_id'] = df_copy.index
     
     # Create the format expected by the duplicate detection system
-    # Using 'page' as row number and 'text' as the combined text
+    # Using 'row_number' as row number and 'text' as the combined text
     processed_df = pd.DataFrame({
-        'page': df_copy['row_id'],
+        'row_number': df_copy['row_id'],
         'text': df_copy['combined_text'],
         'file': file_name
     })
@@ -86,13 +92,15 @@ def find_duplicate_cells_in_tabular_data(
     input_files: List[str],
     similarity_threshold: float = 0.95,
     min_word_count: int = 3,
-    text_columns: List[str] = None,
+    text_columns: List[str] = [],
     output_folder: str = OUTPUT_FOLDER,
     do_initial_clean_dup: bool = DO_INITIAL_TABULAR_DATA_CLEAN,
+    remove_duplicate_rows: bool = REMOVE_DUPLICATE_ROWS,
+    in_excel_tabular_sheets: str = "",
     progress: Progress = Progress(track_tqdm=True)
 ) -> Tuple[pd.DataFrame, List[str], Dict[str, pd.DataFrame]]:
     """
-    Find duplicate cells/text in tabular data files (CSV, XLSX).
+    Find duplicate cells/text in tabular data files (CSV, XLSX, Parquet).
     
     Args:
         input_files (List[str]): List of file paths to analyze
@@ -115,26 +123,49 @@ def find_duplicate_cells_in_tabular_data(
     
     progress(0.1, desc="Loading and processing files...")
     
-    all_data_to_process = []
-    full_data_by_file = {}
-    file_paths = []
+    all_data_to_process = list()
+    full_data_by_file = dict()
+    file_paths = list()    
     
     # Process each file
     for file_path in input_files:
         try:
-            df = read_file(file_path)
-            
-            file_name = os.path.basename(file_path)
-            file_paths.append(file_path)
-            
-            # Convert to analysis format
-            processed_data = convert_tabular_data_to_analysis_format(
-                df, file_name, text_columns
-            )
+            if file_path.endswith('.xlsx') or file_path.endswith('.xls'):
+                temp_df = pd.DataFrame()
+
+                # Try finding each sheet in the given list until a match is found
+                for sheet_name in in_excel_tabular_sheets:
+                    temp_df = read_file(file_path, excel_sheet_name=sheet_name)
+
+                    # If sheet was successfully_loaded
+                    if not temp_df.empty:
+                        file_name = os.path.basename(file_path) + "_" + sheet_name
+                        file_paths.append(file_path)
+                        
+                        # Convert to analysis format
+                        processed_data = convert_tabular_data_to_analysis_format(
+                            temp_df, file_name, text_columns
+                        )
+                        
+                        if processed_data:
+                            all_data_to_process.extend(processed_data)
+                            full_data_by_file[file_name] = processed_data[0][1]
+
+                    temp_df = pd.DataFrame()
+            else:
+                temp_df = read_file(file_path)
             
-            if processed_data:
-                all_data_to_process.extend(processed_data)
-                full_data_by_file[file_name] = processed_data[0][1]
+                file_name = os.path.basename(file_path)
+                file_paths.append(file_path)
+                
+                # Convert to analysis format
+                processed_data = convert_tabular_data_to_analysis_format(
+                    temp_df, file_name, text_columns
+                )
+                
+                if processed_data:
+                    all_data_to_process.extend(processed_data)
+                    full_data_by_file[file_name] = processed_data[0][1]
             
         except Exception as e:
             print(f"Error processing {file_path}: {e}")
@@ -147,6 +178,8 @@ def find_duplicate_cells_in_tabular_data(
     
     # Combine all data
     combined_df = pd.concat([data[1] for data in all_data_to_process], ignore_index=True)
+
+    combined_df = combined_df.drop_duplicates(subset=['row_number', 'file'])
     
     progress(0.3, desc="Cleaning and preparing text...")
     
@@ -188,9 +221,9 @@ def find_duplicate_cells_in_tabular_data(
         
         results_data.append({
             'File1': row1_data['file'],
-            'Row1': int(row1_data['page']),
+            'Row1': int(row1_data['row_number']),
             'File2': row2_data['file'],
-            'Row2': int(row2_data['page']),
+            'Row2': int(row2_data['row_number']),
             'Similarity_Score': round(similarity, 3),
             'Text1': row1_data['text'][:200] + '...' if len(row1_data['text']) > 200 else row1_data['text'],
             'Text2': row2_data['text'][:200] + '...' if len(row2_data['text']) > 200 else row2_data['text'],
@@ -204,13 +237,13 @@ def find_duplicate_cells_in_tabular_data(
     progress(0.9, desc="Saving results...")
     
     # Save results
-    output_paths = save_tabular_duplicate_results(results_df, output_folder, file_paths, file_replaced_index=0)
+    output_paths = save_tabular_duplicate_results(results_df, output_folder, file_paths, remove_duplicate_rows=remove_duplicate_rows, in_excel_tabular_sheets=in_excel_tabular_sheets)
     
     gr.Info(f"Found {len(results_df)} duplicate cell matches")
     
     return results_df, output_paths, full_data_by_file
 
-def save_tabular_duplicate_results(results_df: pd.DataFrame, output_folder: str, file_paths: List[str], file_replaced_index: int = 0) -> List[str]:
+def save_tabular_duplicate_results(results_df: pd.DataFrame, output_folder: str, file_paths: List[str], remove_duplicate_rows: bool = REMOVE_DUPLICATE_ROWS, in_excel_tabular_sheets: List[str] = []) -> List[str]:
     """
     Save tabular duplicate detection results to files.
     
@@ -218,52 +251,163 @@ def save_tabular_duplicate_results(results_df: pd.DataFrame, output_folder: str,
         results_df (pd.DataFrame): Results DataFrame
         output_folder (str): Output folder path
         file_paths (List[str]): List of file paths
-        file_replaced_index (int): Index of the file to replace with duplicate rows removed
-            (0 is the first file in the list)
+        remove_duplicate_rows (bool): Whether to remove duplicate rows
+        in_excel_tabular_sheets (str): Name of the Excel sheet to save the results to
     Returns:
         List[str]: List of output file paths
     """
-    output_paths = []
+    output_paths = list()
     output_folder_path = Path(output_folder)
     output_folder_path.mkdir(exist_ok=True)
     
     if results_df.empty:
         print("No duplicate matches to save.")
-        return []
+        return list()
     
     # Save main results
     results_file = output_folder_path / 'tabular_duplicate_results.csv'
     results_df.to_csv(results_file, index=False, encoding="utf-8-sig")
     output_paths.append(str(results_file))
     
+    # Group results by original file to handle Excel files properly
+    excel_files_processed = dict() # Track which Excel files have been processed
+    
     # Save per-file duplicate lists
-    for file_name, group in results_df.groupby('File1'):
-        file_stem = Path(file_name).stem
-        duplicate_rows_file = output_folder_path / f"{file_stem}_duplicate_rows.csv"
-        
-        # Get unique row numbers to remove
-        rows_to_remove = sorted(group['Row1'].unique())
-        duplicate_df = pd.DataFrame({'Row_to_Remove': rows_to_remove})
-        duplicate_df.to_csv(duplicate_rows_file, index=False)
-        output_paths.append(str(duplicate_rows_file))
-
-        # Save also original file (first file in list) with duplicate rows removed
-        file_path = file_paths[file_replaced_index]
-        file_base_name = os.path.basename(file_path)
-        df = read_file(file_path)
-        df_cleaned = df.drop(index=rows_to_remove).reset_index(drop=True)
-
-        output_path = os.path.join(output_folder, f"{file_base_name}_deduplicated.csv")
-        df_cleaned.to_csv(output_path, index=False, encoding="utf-8-sig")
-        
-        output_paths.append(str(output_path))
+    for file_name, group in results_df.groupby('File2'):
+        # Check for matches with original file names
+        for original_file in file_paths:
+            original_file_name = os.path.basename(original_file)
+
+            if original_file_name in file_name:
+                original_file_extension = os.path.splitext(original_file)[-1]
+                if original_file_extension in ['.xlsx', '.xls']:                 
+                    
+                    # Split the string using a regex to handle both .xlsx_ and .xls_ delimiters
+                    # The regex r'\.xlsx_|\.xls_' correctly matches either ".xlsx_" or ".xls_" as a delimiter.
+                    parts = re.split(r'\.xlsx_|\.xls_', os.path.basename(file_name))
+                    # The sheet name is the last part after splitting
+                    file_sheet_name = parts[-1]
+
+                    file_path = original_file
+                    
+                    # Initialize Excel file tracking if not already done
+                    if file_path not in excel_files_processed:
+                        excel_files_processed[file_path] = {
+                            'sheets_data': dict(),
+                            'all_sheets': list(),
+                            'processed_sheets': set()
+                        }
+                    
+                    # Read the original Excel file to get all sheet names
+                    if not excel_files_processed[file_path]['all_sheets']:
+                        try:
+                            excel_file = pd.ExcelFile(file_path)
+                            excel_files_processed[file_path]['all_sheets'] = excel_file.sheet_names
+                        except Exception as e:
+                            print(f"Error reading Excel file {file_path}: {e}")
+                            continue
+                    
+                    # Read the current sheet
+                    df = read_file(file_path, excel_sheet_name=file_sheet_name)
+                    
+                    # Create duplicate rows file for this sheet
+                    file_stem = Path(file_name).stem
+                    duplicate_rows_file = output_folder_path / f"{file_stem}_{file_sheet_name}_duplicate_rows.csv"
+                    
+                    # Get unique row numbers to remove
+                    rows_to_remove = sorted(group['Row2'].unique())
+                    duplicate_df = pd.DataFrame({'Row_to_Remove': rows_to_remove})
+                    duplicate_df.to_csv(duplicate_rows_file, index=False)
+                    output_paths.append(str(duplicate_rows_file))
+                    
+                    # Process the sheet data
+                    df_cleaned = df.copy()
+                    df_cleaned["duplicated"] = False
+                    df_cleaned.loc[rows_to_remove, "duplicated"] = True
+                    if remove_duplicate_rows:
+                        df_cleaned = df_cleaned.drop(index=rows_to_remove)
+                    
+                    # Store the processed sheet data
+                    excel_files_processed[file_path]['sheets_data'][file_sheet_name] = df_cleaned
+                    excel_files_processed[file_path]['processed_sheets'].add(file_sheet_name)
+                    
+                else:
+                    file_sheet_name = ""
+                    file_path = original_file
+                    print("file_path after match:", file_path)
+                    file_base_name = os.path.basename(file_path)
+                    df = read_file(file_path)
+
+                    file_stem = Path(file_name).stem
+                    duplicate_rows_file = output_folder_path / f"{file_stem}_duplicate_rows.csv"
+                    
+                    # Get unique row numbers to remove
+                    rows_to_remove = sorted(group['Row2'].unique())
+                    duplicate_df = pd.DataFrame({'Row_to_Remove': rows_to_remove})
+                    duplicate_df.to_csv(duplicate_rows_file, index=False)
+                    output_paths.append(str(duplicate_rows_file))
+
+                    df_cleaned = df.copy()
+                    df_cleaned["duplicated"] = False
+                    df_cleaned.loc[rows_to_remove, "duplicated"] = True
+                    if remove_duplicate_rows:
+                        df_cleaned = df_cleaned.drop(index=rows_to_remove)
+
+                    file_ext = os.path.splitext(file_name)[-1]
+
+                    if file_ext in ['.parquet']:
+                        output_path = os.path.join(output_folder, f"{file_base_name}_deduplicated.parquet")
+                        df_cleaned.to_parquet(output_path, index=False)
+                    else:
+                        output_path = os.path.join(output_folder, f"{file_base_name}_deduplicated.csv")
+                        df_cleaned.to_csv(output_path, index=False, encoding="utf-8-sig")
+                    
+                    output_paths.append(str(output_path))
+                break
+    
+    # Process Excel files to create complete deduplicated files
+    for file_path, file_data in excel_files_processed.items():
+        try:
+            # Create output filename
+            file_base_name = os.path.splitext(os.path.basename(file_path))[0]
+            file_ext = os.path.splitext(file_path)[-1]
+            output_path = os.path.join(output_folder, f"{file_base_name}_deduplicated{file_ext}")
+            
+            # Create Excel writer
+            with pd.ExcelWriter(output_path, engine='openpyxl') as writer:
+                # Write all sheets
+                for sheet_name in file_data['all_sheets']:
+                    if sheet_name in file_data['processed_sheets']:
+                        # Use the processed (deduplicated) version
+                        file_data['sheets_data'][sheet_name].to_excel(
+                            writer, 
+                            sheet_name=sheet_name, 
+                            index=False
+                        )
+                    else:
+                        # Use the original sheet (no duplicates found)
+                        original_df = read_file(file_path, excel_sheet_name=sheet_name)
+                        original_df.to_excel(
+                            writer, 
+                            sheet_name=sheet_name, 
+                            index=False
+                        )
+            
+            output_paths.append(str(output_path))
+            print(f"Created deduplicated Excel file: {output_path}")
+            
+        except Exception as e:
+            print(f"Error creating deduplicated Excel file for {file_path}: {e}")
+            continue
     
     return output_paths
 
 def remove_duplicate_rows_from_tabular_data(
     file_path: str,
     duplicate_rows: List[int],
-    output_folder: str = OUTPUT_FOLDER
+    output_folder: str = OUTPUT_FOLDER,
+    in_excel_tabular_sheets: List[str] = [],
+    remove_duplicate_rows: bool = REMOVE_DUPLICATE_ROWS
 ) -> str:
     """
     Remove duplicate rows from a tabular data file.
@@ -272,13 +416,14 @@ def remove_duplicate_rows_from_tabular_data(
         file_path (str): Path to the input file
         duplicate_rows (List[int]): List of row indices to remove
         output_folder (str): Output folder for cleaned file
-    
+        in_excel_tabular_sheets (str): Name of the Excel sheet to save the results to
+        remove_duplicate_rows (bool): Whether to remove duplicate rows
     Returns:
         str: Path to the cleaned file
     """
     try:
         # Load the file
-        df = read_file(file_path)
+        df = read_file(file_path, excel_sheet_name=in_excel_tabular_sheets if in_excel_tabular_sheets else "")
         
         # Remove duplicate rows (0-indexed)
         df_cleaned = df.drop(index=duplicate_rows).reset_index(drop=True)
@@ -286,12 +431,12 @@ def remove_duplicate_rows_from_tabular_data(
         # Save cleaned file
         file_name = os.path.basename(file_path)
         file_stem = os.path.splitext(file_name)[0]
-        file_ext = os.path.splitext(file_name)[1]
+        file_ext = os.path.splitext(file_name)[-1]
         
         output_path = os.path.join(output_folder, f"{file_stem}_deduplicated{file_ext}")
         
         if file_ext in ['.xlsx', '.xls']:
-            df_cleaned.to_excel(output_path, index=False)
+            df_cleaned.to_excel(output_path, index=False, sheet_name=in_excel_tabular_sheets if in_excel_tabular_sheets else [])
         elif file_ext in ['.parquet']:
             df_cleaned.to_parquet(output_path, index=False)
         else:
@@ -307,9 +452,11 @@ def run_tabular_duplicate_analysis(
     files: List[str],
     threshold: float,
     min_words: int,
-    text_columns: List[str] = None,
+    text_columns: List[str] = [],
     output_folder: str = OUTPUT_FOLDER,
     do_initial_clean_dup: bool = DO_INITIAL_TABULAR_DATA_CLEAN,
+    remove_duplicate_rows: bool = REMOVE_DUPLICATE_ROWS,
+    in_excel_tabular_sheets: List[str] = [],
     progress: Progress = Progress(track_tqdm=True)
 ) -> Tuple[pd.DataFrame, List[str], Dict[str, pd.DataFrame]]:
     """
@@ -330,26 +477,37 @@ def run_tabular_duplicate_analysis(
         input_files=files,
         similarity_threshold=threshold,
         min_word_count=min_words,
-        text_columns=text_columns,
+        text_columns=text_columns if text_columns else [],
         output_folder=output_folder,
         do_initial_clean_dup=do_initial_clean_dup,
-        progress=progress
+        in_excel_tabular_sheets=in_excel_tabular_sheets if in_excel_tabular_sheets else [],
+        remove_duplicate_rows=remove_duplicate_rows
     )
 
 
 
 # Function to update column choices when files are uploaded
-def update_tabular_column_choices(files):
+def update_tabular_column_choices(files, in_excel_tabular_sheets: List[str] = []):
     if not files:
         return gr.update(choices=[])
     
     all_columns = set()
     for file in files:
         try:
-            df = read_file(file.name)
-            
+            file_extension = os.path.splitext(file.name)[-1]
+            if file_extension in ['.xlsx', '.xls']:
+                for sheet_name in in_excel_tabular_sheets:
+                    df = read_file(file.name, excel_sheet_name=sheet_name)
+                    text_cols = df.select_dtypes(include=['object', 'string']).columns.tolist()
+                    all_columns.update(text_cols)
+            else:
+                df = read_file(file.name)
+                text_cols = df.select_dtypes(include=['object', 'string']).columns.tolist()
+                all_columns.update(text_cols)
+
             # Get text columns
             text_cols = df.select_dtypes(include=['object', 'string']).columns.tolist()
+
             all_columns.update(text_cols)
         except Exception as e:
             print(f"Error reading {file.name}: {e}")
@@ -358,26 +516,59 @@ def update_tabular_column_choices(files):
     return gr.Dropdown(choices=sorted(list(all_columns)))
 
 # Function to handle tabular duplicate detection
-def run_tabular_duplicate_detection(files, threshold, min_words, text_columns, output_folder: str = OUTPUT_FOLDER, do_initial_clean_dup: bool = DO_INITIAL_TABULAR_DATA_CLEAN):
+def run_tabular_duplicate_detection(files, threshold, min_words, text_columns, output_folder: str = OUTPUT_FOLDER, do_initial_clean_dup: bool = DO_INITIAL_TABULAR_DATA_CLEAN, in_excel_tabular_sheets: List[str] = [], remove_duplicate_rows: bool = REMOVE_DUPLICATE_ROWS):
     if not files:
-        return pd.DataFrame(), [], gr.Dropdown(choices=[])
+        print("No files uploaded")
+        return pd.DataFrame(), [], gr.Dropdown(choices=[]), 0, "deduplicate"
+
+    start_time = time.time()
+
+    task_textbox = "deduplicate"
+
+    # If output folder doesn't end with a forward slash, add one
+    if not output_folder.endswith('/'): output_folder = output_folder + '/'
     
-    file_paths = [f.name for f in files]
+    file_paths = []
+    if isinstance(files, str):
+        # If 'files' is a single string, treat it as a list with one element
+        file_paths.append(files)
+    elif isinstance(files, list):
+        # If 'files' is a list, iterate through its elements
+        for f_item in files:
+            if isinstance(f_item, str):
+                # If an element is a string, it's a direct file path
+                file_paths.append(f_item)
+            elif hasattr(f_item, 'name'):
+                # If an element has a '.name' attribute (e.g., a Gradio File object), use its name
+                file_paths.append(f_item.name)
+            else:
+                # Log a warning for unexpected element types within the list
+                print(f"Warning: Skipping an element in 'files' list that is neither a string nor has a '.name' attribute: {type(f_item)}")
+    elif hasattr(files, 'name'):
+        # Handle the case where a single file object (e.g., gr.File) is passed directly, not in a list
+        file_paths.append(files.name)
+    else:
+        # Raise an error for any other unexpected type of the 'files' argument itself
+        raise TypeError(f"Unexpected type for 'files' argument: {type(files)}. Expected str, list of str/file objects, or a single file object.")
+
     results_df, output_paths, full_data = run_tabular_duplicate_analysis(
         files=file_paths,
         threshold=threshold,
         min_words=min_words,
-        text_columns=text_columns if text_columns else None,
+        text_columns=text_columns if text_columns else [],
         output_folder=output_folder,
-        do_initial_clean_dup=do_initial_clean_dup
+        do_initial_clean_dup=do_initial_clean_dup,
+        in_excel_tabular_sheets=in_excel_tabular_sheets if in_excel_tabular_sheets else None,
+        remove_duplicate_rows=remove_duplicate_rows
     )
-
-    print("output_paths:", output_paths)
     
     # Update file choices for cleaning
     file_choices = list(set([f for f in file_paths]))
+
+    end_time = time.time()
+    processing_time = round(end_time - start_time, 2)
     
-    return results_df, output_paths, gr.Dropdown(choices=file_choices)
+    return results_df, output_paths, gr.Dropdown(choices=file_choices), processing_time, task_textbox
 
 # Function to handle row selection for preview
 def handle_tabular_row_selection(results_df, evt:gr.SelectData):
@@ -398,12 +589,12 @@ def handle_tabular_row_selection(results_df, evt:gr.SelectData):
     return selected_index, row['Text1'], row['Text2']
 
 # Function to clean duplicates from selected file
-def clean_tabular_duplicates(file_name, results_df, output_folder):
+def clean_tabular_duplicates(file_name, results_df, output_folder, in_excel_tabular_sheets: str = "", remove_duplicate_rows: bool = REMOVE_DUPLICATE_ROWS):
     if not file_name or results_df.empty:
         return None
     
     # Get duplicate rows for this file
-    file_duplicates = results_df[results_df['File1'] == file_name]['Row1'].tolist()
+    file_duplicates = results_df[results_df['File2'] == file_name]['Row2'].tolist()
     
     if not file_duplicates:
         return None
@@ -414,7 +605,9 @@ def clean_tabular_duplicates(file_name, results_df, output_folder):
         cleaned_file = remove_duplicate_rows_from_tabular_data(
             file_path=file_name,
             duplicate_rows=file_duplicates,
-            output_folder=output_folder
+            output_folder=output_folder,
+            in_excel_tabular_sheets=in_excel_tabular_sheets,
+            remove_duplicate_rows=remove_duplicate_rows
         )
         return cleaned_file
     except Exception as e:

tools/helper_functions.py

@@ -10,7 +10,6 @@ from typing import List
 from math import ceil
 from gradio_image_annotation import image_annotator
 from tools.config import CUSTOM_HEADER_VALUE, CUSTOM_HEADER, OUTPUT_FOLDER, INPUT_FOLDER, SESSION_OUTPUT_FOLDER, AWS_USER_POOL_ID, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_INPUT_SUBFOLDER, TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_OUTPUT_SUBFOLDER, TEXTRACT_JOBS_S3_LOC, TEXTRACT_JOBS_LOCAL_LOC, SELECTABLE_TEXT_EXTRACT_OPTION, TESSERACT_TEXT_EXTRACT_OPTION, TEXTRACT_TEXT_EXTRACT_OPTION, NO_REDACTION_PII_OPTION, AWS_PII_OPTION, MAPPED_LANGUAGE_CHOICES, LANGUAGE_CHOICES, textract_language_choices, aws_comprehend_language_choices, DEFAULT_LANGUAGE
-# from tools.load_spacy_model_custom_recognisers import nlp_analyser
 
 def _get_env_list(env_var_name: str) -> List[str]:
     """Parses a comma-separated environment variable into a list of strings."""
@@ -147,14 +146,21 @@ def detect_file_type(filename:str):
     elif filename.endswith('.docx'): return 'docx'
     else: raise ValueError("Unsupported file type.")
 
-def read_file(filename:str):
+def read_file(filename:str, excel_sheet_name: str = ""):
     """Read the file based on its detected type."""
     file_type = detect_file_type(filename)
     
     if file_type == 'csv':
         return pd.read_csv(filename, low_memory=False)
     elif file_type == 'xlsx':
-        return pd.read_excel(filename)
+        if excel_sheet_name:
+            try:
+                return pd.read_excel(filename, sheet_name=excel_sheet_name)
+            except Exception as e:
+                print(f"Error reading {filename} with sheet name {excel_sheet_name}: {e}")
+                return pd.DataFrame()
+        else:
+            return pd.read_excel(filename)
     elif file_type == 'parquet':
         return pd.read_parquet(filename)
 

tools/load_spacy_model_custom_recognisers.py

@@ -11,10 +11,13 @@ import re
 import os
 import requests
 import gradio as gr
-from tools.config import DEFAULT_LANGUAGE, TESSERACT_DATA_FOLDER
+from tools.config import DEFAULT_LANGUAGE, TESSERACT_DATA_FOLDER, CUSTOM_ENTITIES
+from tools.helper_functions import _get_env_list
 
 score_threshold = 0.001
-custom_entities = ["TITLES", "UKPOSTCODE", "STREETNAME", "CUSTOM"]
+
+if CUSTOM_ENTITIES: CUSTOM_ENTITIES = _get_env_list(CUSTOM_ENTITIES)
+custom_entities = CUSTOM_ENTITIES
 
 # Create a class inheriting from SpacyNlpEngine
 class LoadedSpacyNlpEngine(SpacyNlpEngine):

tools/textract_batch_call.py

@@ -1,17 +1,17 @@
 import boto3
-import time
 import os
 import pandas as pd
 import json
 import logging
 import datetime
+import pymupdf
 import gradio as gr
 from gradio import FileData
 from typing import List
 from io import StringIO
 from urllib.parse import urlparse
 from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError, TokenRetrievalError
-from tools.config import TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, OUTPUT_FOLDER, AWS_REGION, DOCUMENT_REDACTION_BUCKET, LOAD_PREVIOUS_TEXTRACT_JOBS_S3, TEXTRACT_JOBS_S3_LOC, TEXTRACT_JOBS_LOCAL_LOC, TEXTRACT_JOBS_S3_INPUT_LOC, RUN_AWS_FUNCTIONS, INPUT_FOLDER, DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS
+from tools.config import TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET, OUTPUT_FOLDER, AWS_REGION, DOCUMENT_REDACTION_BUCKET, LOAD_PREVIOUS_TEXTRACT_JOBS_S3, TEXTRACT_JOBS_S3_LOC, TEXTRACT_JOBS_LOCAL_LOC, RUN_AWS_FUNCTIONS, INPUT_FOLDER, DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS
 from tools.aws_functions import download_file_from_s3
 from tools.file_conversion import get_input_file_names
 from tools.helper_functions import get_file_name_without_type
@@ -59,6 +59,7 @@ def analyse_document_with_textract_api(
 
     # This is a variable that is written to logs to indicate that a Textract API call was made
     is_a_textract_api_call = True
+    task_textbox = "textract"
 
     # Keep only latest pdf path if it's a list
     if isinstance(local_pdf_path, list):
@@ -67,6 +68,23 @@ def analyse_document_with_textract_api(
     if not os.path.exists(local_pdf_path):
         raise FileNotFoundError(f"Input document not found {local_pdf_path}")
 
+    file_extension = os.path.splitext(local_pdf_path)[1].lower()
+
+    # Load pdf to get page count if not provided
+    if not total_document_page_count and file_extension in ['.pdf']:
+        print("Page count not provided. Loading PDF to get page count")
+        try:
+            pymupdf_doc = pymupdf.open(local_pdf_path)
+            total_document_page_count = pymupdf_doc.page_count
+            pymupdf_doc.close()
+            print("Page count:", total_document_page_count)
+        except Exception as e:
+            print("Failed to load PDF to get page count:", e, "setting page count to 1")
+            total_document_page_count = 1
+            #raise Exception(f"Failed to load PDF to get page count: {e}")
+    else:
+        total_document_page_count = 1
+
     if not os.path.exists(local_output_dir):
         os.makedirs(local_output_dir)
         log_message = f"Created local output directory: {local_output_dir}"
@@ -96,18 +114,32 @@ def analyse_document_with_textract_api(
         #logging.error(log_message)
         raise
 
-    # If job_df is not empty
+    # Filter job_df to include rows only where the analysis date is after the current date - DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS
     if not job_df.empty:
+        job_df = job_df.loc[job_df["job_date_time"] > (datetime.datetime.now() - datetime.timedelta(days=DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS)),:]
+
+    # If job_df is not empty
+    if not job_df.empty:        
+
         if "file_name" in job_df.columns:
             matching_job_id_file_names = job_df.loc[(job_df["file_name"] == pdf_filename) & (job_df["signature_extraction"].astype(str) == str(analyse_signatures)), "file_name"]
+            matching_job_id_file_names_dates = job_df.loc[(job_df["file_name"] == pdf_filename) & (job_df["signature_extraction"].astype(str) == str(analyse_signatures)), "job_date_time"]
+            matching_job_id = job_df.loc[(job_df["file_name"] == pdf_filename) & (job_df["signature_extraction"].astype(str) == str(analyse_signatures)), "job_id"]
+            matching_handwrite_signature = job_df.loc[(job_df["file_name"] == pdf_filename) & (job_df["signature_extraction"].astype(str) == str(analyse_signatures)), "signature_extraction"]
 
-            if len(matching_job_id_file_names) > 0:
-                    raise Exception("Existing Textract outputs found. No need to re-analyse. Please download existing results from the list")
+            if len(matching_job_id) > 0:
+                pass
+            else:
+                matching_job_id = "unknown_job_id"
+
+            if len(matching_job_id_file_names) > 0 and len(matching_handwrite_signature) > 0:
+                out_message = f"Existing Textract outputs found for file {pdf_filename} from date {matching_job_id_file_names_dates.iloc[0]}. No need to re-analyse. Please download existing results from the list with job ID {matching_job_id.iloc[0]}"
+                gr.Warning(out_message)
+                raise Exception(out_message)
 
     # --- 2. Start Textract Document Analysis ---
     message = "Starting Textract document analysis job..."
     print(message)
-    #logging.info("Starting Textract document analysis job...")
 
     try:
         if "Extract signatures" in analyse_signatures:
@@ -143,19 +175,12 @@ def analyse_document_with_textract_api(
                     'S3Bucket': s3_bucket_name,
                     'S3Prefix': s3_output_prefix
                 }
-                # Optional: Add NotificationChannel for SNS topic notifications
-                # NotificationChannel={
-                #     'SNSTopicArn': 'YOUR_SNS_TOPIC_ARN',
-                #     'RoleArn': 'YOUR_IAM_ROLE_ARN_FOR_TEXTRACT_TO_ACCESS_SNS'
-                # }
             )
             job_type="document_text_detection"
 
         job_id = response['JobId']
         print(f"Textract job started with JobId: {job_id}")
-        #logging.info(f"Textract job started with JobId: {job_id}")
 
-        # Write job_id to memory
         # Prepare CSV in memory
         log_csv_key_location = f"{s3_output_prefix}/textract_document_jobs.csv"
         job_location_full = f"s3://{s3_bucket_name}/{s3_output_prefix}/{job_id}/"
@@ -166,12 +191,16 @@ def analyse_document_with_textract_api(
             'file_name': pdf_filename,
             'job_type': job_type,
             'signature_extraction':analyse_signatures,
-            #'s3_location': job_location_full,
-            'job_date_time': datetime.datetime.now()
+            'job_date_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         }])
 
         # File path
         log_file_path = os.path.join(local_output_dir, "textract_document_jobs.csv")
+        log_file_path_job_id = os.path.join(local_output_dir, pdf_filename + "_textract_document_jobs_job_id.txt")
+
+        # Write latest job ID to local text file
+        with open(log_file_path_job_id, 'w') as f:
+            f.write(job_id)
 
         # Check if file exists
         file_exists = os.path.exists(log_file_path)
@@ -198,7 +227,7 @@ def analyse_document_with_textract_api(
     successful_job_number += 1
     total_number_of_textract_page_calls = total_document_page_count
 
-    return f"Textract analysis job submitted, job ID:{job_id}", job_id, job_type, successful_job_number, is_a_textract_api_call, total_number_of_textract_page_calls
+    return f"Textract analysis job submitted, job ID:{job_id}", job_id, job_type, successful_job_number, is_a_textract_api_call, total_number_of_textract_page_calls, task_textbox
 
 def return_job_status(job_id:str,
                      response:dict,
@@ -467,13 +496,19 @@ def poll_whole_document_textract_analysis_progress_and_download(
             progress(0.5, "Document analysis task outputs found. Downloading from S3")                 
 
             # If job_df is not empty
+
+            # if not job_df.empty:
+            #     job_df = job_df.loc[job_df["job_date_time"] > (datetime.datetime.now() - datetime.timedelta(days=DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS)),:]
+
             if not job_df.empty:
                 if "file_name" in job_df.columns:
                     matching_job_id_file_names = job_df.loc[job_df["job_id"] == job_id, "file_name"]
 
                     if pdf_filename and not matching_job_id_file_names.empty:
                         if pdf_filename == matching_job_id_file_names.iloc[0]:
-                            raise Exception("Existing Textract outputs found. No need to re-download.")
+                            out_message = f"Existing Textract outputs found for file {pdf_filename}. No need to re-download."
+                            gr.Warning(out_message)
+                            raise Exception(out_message)
 
                     if not matching_job_id_file_names.empty:
                         pdf_filename = matching_job_id_file_names.iloc[0]