Major update. General code revision. Improved config variables. Dataframe based review frame now includes text, items can be searched and excluded. Costs now estimated. Option for adding cost codes added. Option to extract text only.

app.py

@@ -1,33 +1,27 @@
 import os
-import socket
-
-# By default TLDExtract will try to pull files from the internet. I have instead downloaded this file locally to avoid the requirement for an internet connection.
-#os.environ['TLDEXTRACT_CACHE'] = 'tld/.tld_set_snapshot'
-
-import gradio as gr
 import pandas as pd
-from datetime import datetime
+import gradio as gr
 from gradio_image_annotation import image_annotator
-from gradio_image_annotation.image_annotator import AnnotatedImageData
 
-from tools.config import output_folder, RUN_DIRECT_MODE, MAX_QUEUE_SIZE, DEFAULT_CONCURRENCY_LIMIT, MAX_FILE_SIZE, GRADIO_SERVER_PORT, ROOT_PATH, GET_DEFAULT_ALLOW_LIST, DEFAULT_ALLOW_LIST_PATH
-from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, put_columns_in_df, get_connection_params, reveal_feedback_buttons, custom_regex_load, reset_state_vars, load_in_default_allow_list, tesseract_ocr_option, text_ocr_option, textract_option, local_pii_detector, aws_pii_detector, reset_review_vars, merge_csv_files, load_all_output_files, update_dataframe
-from tools.aws_functions import upload_file_to_s3, download_file_from_s3, RUN_AWS_FUNCTIONS, bucket_name
+from tools.config import OUTPUT_FOLDER, INPUT_FOLDER, RUN_DIRECT_MODE, MAX_QUEUE_SIZE, DEFAULT_CONCURRENCY_LIMIT, MAX_FILE_SIZE, GRADIO_SERVER_PORT, ROOT_PATH, GET_DEFAULT_ALLOW_LIST, ALLOW_LIST_PATH, S3_ALLOW_LIST_PATH, FEEDBACK_LOGS_FOLDER, ACCESS_LOGS_FOLDER, USAGE_LOGS_FOLDER, TESSERACT_FOLDER, POPPLER_FOLDER, REDACTION_LANGUAGE, GET_COST_CODES, COST_CODES_PATH, S3_COST_CODES_PATH, ENFORCE_COST_CODES, DISPLAY_FILE_NAMES_IN_LOGS, SHOW_COSTS, RUN_AWS_FUNCTIONS, DOCUMENT_REDACTION_BUCKET, SHOW_BULK_TEXTRACT_CALL_OPTIONS
+from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, put_columns_in_df, get_connection_params, reveal_feedback_buttons, custom_regex_load, reset_state_vars, load_in_default_allow_list, tesseract_ocr_option, text_ocr_option, textract_option, local_pii_detector, aws_pii_detector, no_redaction_option, reset_review_vars, merge_csv_files, load_all_output_files, update_dataframe, check_for_existing_textract_file, load_in_default_cost_codes, enforce_cost_codes, calculate_aws_costs, calculate_time_taken
+from tools.aws_functions import upload_file_to_s3, download_file_from_s3
 from tools.file_redaction import choose_and_run_redactor
-from tools.file_conversion import prepare_image_or_pdf, get_input_file_names
-from tools.redaction_review import apply_redactions, modify_existing_page_redactions, decrease_page, increase_page, update_annotator, update_zoom, update_entities_df_recogniser_entities, update_entities_df_page, update_entities_df_text, df_select_callback, convert_df_to_xfdf, convert_xfdf_to_dataframe, reset_dropdowns, exclude_selected_items_from_redaction, undo_last_removal
+from tools.file_conversion import prepare_image_or_pdf, get_input_file_names, convert_review_df_to_annotation_json
+from tools.redaction_review import apply_redactions_to_review_df_and_files, update_all_page_annotation_object_based_on_previous_page, decrease_page, increase_page, update_annotator_object_and_filter_df, update_entities_df_recogniser_entities, update_entities_df_page, update_entities_df_text, df_select_callback, convert_df_to_xfdf, convert_xfdf_to_dataframe, reset_dropdowns, exclude_selected_items_from_redaction, undo_last_removal, update_selected_review_df_row_colour, update_all_entity_df_dropdowns, df_select_callback_cost, update_other_annotator_number_from_current, update_annotator_page_from_review_df
 from tools.data_anonymise import anonymise_data_files
 from tools.auth import authenticate_user
 from tools.load_spacy_model_custom_recognisers import custom_entities
 from tools.custom_csvlogger import CSVLogger_custom
 from tools.find_duplicate_pages import identify_similar_pages
 
-today_rev = datetime.now().strftime("%Y%m%d")
+# Suppress downcasting warnings
+pd.set_option('future.no_silent_downcasting', True)
 
-add_folder_to_path("tesseract/")
-add_folder_to_path("poppler/poppler-24.02.0/Library/bin/")
+add_folder_to_path(TESSERACT_FOLDER)
+add_folder_to_path(POPPLER_FOLDER)
 
-ensure_output_folder_exists()
+ensure_output_folder_exists(OUTPUT_FOLDER)
 
 chosen_comprehend_entities = ['BANK_ACCOUNT_NUMBER','BANK_ROUTING','CREDIT_DEBIT_NUMBER','CREDIT_DEBIT_CVV','CREDIT_DEBIT_EXPIRY','PIN','EMAIL','ADDRESS','NAME','PHONE', 'PASSPORT_NUMBER','DRIVER_ID', 'USERNAME','PASSWORD', 'IP_ADDRESS','MAC_ADDRESS', 'LICENSE_PLATE','VEHICLE_IDENTIFICATION_NUMBER','UK_NATIONAL_INSURANCE_NUMBER', 'INTERNATIONAL_BANK_ACCOUNT_NUMBER','SWIFT_CODE','UK_NATIONAL_HEALTH_SERVICE_NUMBER']
 
@@ -42,12 +36,7 @@ chosen_redact_entities = ["TITLES", "PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "
 
 full_entity_list = ["TITLES", "PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "STREETNAME", "UKPOSTCODE", 'CREDIT_CARD', 'CRYPTO', 'DATE_TIME', 'IBAN_CODE', 'IP_ADDRESS', 'NRP', 'LOCATION', 'MEDICAL_LICENSE', 'URL', 'UK_NHS', 'CUSTOM', 'CUSTOM_FUZZY']
 
-language = 'en'
-
-host_name = socket.gethostname()
-feedback_logs_folder = 'feedback/' + today_rev + '/' + host_name + '/'
-access_logs_folder = 'logs/' + today_rev + '/' + host_name + '/'
-usage_logs_folder = 'usage/' + today_rev + '/' + host_name + '/'
+log_file_name = 'log.csv'
 
 file_input_height = 200
 
@@ -71,13 +60,14 @@ with app:
     pdf_doc_state = gr.State([])    
     all_image_annotations_state = gr.State([])
 
-    all_line_level_ocr_results_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"),  label="all_line_level_ocr_results_df", visible=False, type="pandas")
-    all_decision_process_table_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"),  label="all_decision_process_table", visible=False, type="pandas")
-    review_file_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="review_file_df", visible=False, type="pandas")
+    all_line_level_ocr_results_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"),  label="all_line_level_ocr_results_df", visible=False, type="pandas", wrap=True)
+    all_decision_process_table_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"),  label="all_decision_process_table", visible=False, type="pandas", wrap=True)
+    review_file_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="review_file_df", visible=False, type="pandas", wrap=True)
 
     session_hash_state = gr.Textbox(label= "session_hash_state", value="", visible=False)
     s3_output_folder_state = gr.Textbox(label= "s3_output_folder_state", value="", visible=False)
-    output_folder_textbox = gr.Textbox(value = output_folder, label="output_folder_textbox", visible=False)
+    output_folder_textbox = gr.Textbox(value = OUTPUT_FOLDER, label="output_folder_textbox", visible=False)
+    input_folder_textbox = gr.Textbox(value = INPUT_FOLDER, label="input_folder_textbox", visible=False)
 
     first_loop_state = gr.Checkbox(label="first_loop_state", value=True, visible=False)
     second_loop_state = gr.Checkbox(label="second_loop_state", value=False, visible=False)
@@ -88,36 +78,38 @@ with app:
     document_cropboxes = gr.Dropdown(label = "document_cropboxes", value="", allow_custom_value=True,visible=False)
     page_sizes = gr.Dropdown(label = "page_sizes", value="", allow_custom_value=True, visible=False)
     images_pdf_state = gr.Dropdown(label = "images_pdf_list", value="", allow_custom_value=True,visible=False)
+    all_img_details_state = gr.State([])
     
     output_image_files_state = gr.Dropdown(label = "output_image_files_list", value="", allow_custom_value=True,visible=False)
     output_file_list_state = gr.Dropdown(label = "output_file_list", value="", allow_custom_value=True,visible=False)
     text_output_file_list_state = gr.Dropdown(label = "text_output_file_list", value="", allow_custom_value=True,visible=False)
-    log_files_output_list_state = gr.Dropdown(label = "log_files_output_list", value="", allow_custom_value=True,visible=False) 
+    log_files_output_list_state = gr.Dropdown(label = "log_files_output_list", value="", allow_custom_value=True,visible=False)
+    duplication_file_path_outputs_list_state = gr.Dropdown(label = "duplication_file_path_outputs_list", value=[], multiselect=True, allow_custom_value=True,visible=False)
 
     # Backup versions of these objects in case you make a mistake
     backup_review_state = gr.Dataframe(visible=False)
     backup_image_annotations_state = gr.State([])
-    backup_recogniser_entity_dataframe_base = gr.Dataframe(visible=False)
-    
+    backup_recogniser_entity_dataframe_base = gr.Dataframe(visible=False)    
     
     # Logging state
-    log_file_name = 'log.csv'
-
-    feedback_logs_state = gr.Textbox(label= "feedback_logs_state", value=feedback_logs_folder + log_file_name, visible=False)
-    feedback_s3_logs_loc_state = gr.Textbox(label= "feedback_s3_logs_loc_state", value=feedback_logs_folder, visible=False)
-    access_logs_state = gr.Textbox(label= "access_logs_state", value=access_logs_folder + log_file_name, visible=False)
-    access_s3_logs_loc_state = gr.Textbox(label= "access_s3_logs_loc_state", value=access_logs_folder, visible=False)
-    usage_logs_state = gr.Textbox(label= "usage_logs_state", value=usage_logs_folder + log_file_name, visible=False)
-    usage_s3_logs_loc_state = gr.Textbox(label= "usage_s3_logs_loc_state", value=usage_logs_folder, visible=False)
+    feedback_logs_state = gr.Textbox(label= "feedback_logs_state", value=FEEDBACK_LOGS_FOLDER + log_file_name, visible=False)
+    feedback_s3_logs_loc_state = gr.Textbox(label= "feedback_s3_logs_loc_state", value=FEEDBACK_LOGS_FOLDER, visible=False)
+    access_logs_state = gr.Textbox(label= "access_logs_state", value=ACCESS_LOGS_FOLDER + log_file_name, visible=False)
+    access_s3_logs_loc_state = gr.Textbox(label= "access_s3_logs_loc_state", value=ACCESS_LOGS_FOLDER, visible=False)
+    usage_logs_state = gr.Textbox(label= "usage_logs_state", value=USAGE_LOGS_FOLDER + log_file_name, visible=False)
+    usage_s3_logs_loc_state = gr.Textbox(label= "usage_s3_logs_loc_state", value=USAGE_LOGS_FOLDER, visible=False)
 
     session_hash_textbox = gr.Textbox(label= "session_hash_textbox", value="", visible=False)
     textract_metadata_textbox = gr.Textbox(label = "textract_metadata_textbox", value="", visible=False)
     comprehend_query_number = gr.Number(label = "comprehend_query_number", value=0, visible=False)
-
+    textract_query_number = gr.Number(label = "textract_query_number", value=0, visible=False)
+    
     doc_full_file_name_textbox = gr.Textbox(label = "doc_full_file_name_textbox", value="", visible=False)
     doc_file_name_no_extension_textbox = gr.Textbox(label = "doc_full_file_name_textbox", value="", visible=False)
+    blank_doc_file_name_no_extension_textbox_for_logs = gr.Textbox(label = "doc_full_file_name_textbox", value="", visible=False) # Left blank for when user does not want to report file names
     doc_file_name_with_extension_textbox = gr.Textbox(label = "doc_file_name_with_extension_textbox", value="", visible=False)
     doc_file_name_textbox_list = gr.Dropdown(label = "doc_file_name_textbox_list", value="", allow_custom_value=True,visible=False)
+    latest_review_file_path = gr.Textbox(label = "latest_review_file_path", value="", visible=False) # Latest review file path output from redaction
 
     data_full_file_name_textbox = gr.Textbox(label = "data_full_file_name_textbox", value="", visible=False)
     data_file_name_no_extension_textbox = gr.Textbox(label = "data_full_file_name_textbox", value="", visible=False)
@@ -129,9 +121,8 @@ with app:
     text_name_const = gr.Textbox(label="text_name_const", value="text", visible=False)
     page_name_const = gr.Textbox(label="page_name_const", value="page", visible=False)
     
-    estimated_time_taken_number = gr.Number(label = "estimated_time_taken_number", value=0.0, precision=1, visible=False) # This keeps track of the time taken to redact files for logging purposes.
+    actual_time_taken_number = gr.Number(label = "actual_time_taken_number", value=0.0, precision=1, visible=False) # This keeps track of the time taken to redact files for logging purposes.
     annotate_previous_page = gr.Number(value=0, label="Previous page", precision=0, visible=False) # Keeps track of the last page that the annotator was on
-
     s3_logs_output_textbox = gr.Textbox(label="Feedback submission logs", visible=False)
 
     ## Annotator zoom value
@@ -141,31 +132,47 @@ with app:
 
     clear_all_page_redactions = gr.Checkbox(label="clear_all_page_redactions", value=True, visible=False)
     prepare_for_review_bool = gr.Checkbox(label="prepare_for_review_bool", value=True, visible=False)
+    prepare_images_bool_false = gr.Checkbox(label="prepare_images_bool_false", value=False, visible=False)
 
     ## Settings page variables
     default_deny_list_file_name = "default_deny_list.csv"
-    default_deny_list_loc = output_folder + "/" + default_deny_list_file_name    
+    default_deny_list_loc = OUTPUT_FOLDER + "/" + default_deny_list_file_name    
     in_deny_list_text_in = gr.Textbox(value="deny_list", visible=False)
 
     fully_redacted_list_file_name = "default_fully_redacted_list.csv"
-    fully_redacted_list_loc = output_folder + "/" + fully_redacted_list_file_name    
+    fully_redacted_list_loc = OUTPUT_FOLDER + "/" + fully_redacted_list_file_name    
     in_fully_redacted_text_in = gr.Textbox(value="fully_redacted_pages_list", visible=False)
 
     # S3 settings for default allow list load
-    s3_default_bucket = gr.Textbox(label = "Default S3 bucket", value=bucket_name, visible=False)
+    s3_default_bucket = gr.Textbox(label = "Default S3 bucket", value=DOCUMENT_REDACTION_BUCKET, visible=False)
+    s3_default_allow_list_file = gr.Textbox(label = "Default allow list file", value=S3_ALLOW_LIST_PATH, visible=False)
+    default_allow_list_output_folder_location = gr.Textbox(label = "Output default allow list location", value=ALLOW_LIST_PATH, visible=False)
 
-    default_allow_list_file_name = "default_allow_list.csv"
-    default_allow_list_loc = output_folder + "/" + default_allow_list_file_name    
-
-    s3_default_allow_list_file = gr.Textbox(label = "Default allow list file", value=DEFAULT_ALLOW_LIST_PATH, visible=False)
-    default_allow_list_output_folder_location = gr.Textbox(label = "Output default allow list location", value=default_allow_list_loc, visible=False)
+    s3_default_cost_codes_file = gr.Textbox(label = "Default cost centre file", value=S3_COST_CODES_PATH, visible=False)
+    default_cost_codes_output_folder_location = gr.Textbox(label = "Output default cost centre location", value=COST_CODES_PATH, visible=False)
+    enforce_cost_code_textbox = gr.Textbox(label = "Enforce cost code textbox", value=ENFORCE_COST_CODES, visible=False)
 
     # Base dataframe for recognisers that is not modified subsequent to load
-    recogniser_entity_dataframe_base = gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), col_count=3, type="pandas", visible=False, label="recogniser_entity_dataframe_base", show_search="filter", headers=["page", "label", "text"])
+    recogniser_entity_dataframe_base = gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), col_count=3, type="pandas", visible=False, label="recogniser_entity_dataframe_base", show_search="filter", headers=["page", "label", "text"], show_fullscreen_button=True, wrap=True)
 
     # Duplicate page detection
     in_duplicate_pages_text = gr.Textbox(label="in_duplicate_pages_text", visible=False)
-    duplicate_pages_df = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="duplicate_pages_df", visible=False, type="pandas")
+    duplicate_pages_df = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="duplicate_pages_df", visible=False, type="pandas", wrap=True)    
+
+    # Tracking variables for current page (not visible)
+    current_loop_page_number = gr.Number(value=0,precision=0, interactive=False, label = "Last redacted page in document", visible=False)
+    page_break_return = gr.Checkbox(value = False, label="Page break reached", visible=False)
+
+    # Placeholders for elements that may be made visible later below depending on environment variables
+    cost_code_dataframe = gr.Dataframe(value=pd.DataFrame(), type="pandas", visible=False, wrap=True)
+    cost_code_choice_drop = gr.Dropdown(value="", label="Choose cost code for analysis", choices=[], allow_custom_value=True, visible=False)
+
+    textract_output_found_checkbox = gr.Checkbox(value= False, label="Existing Textract output file found", interactive=False, visible=False)
+    total_pdf_page_count = gr.Number(label = "Total page count", value=0, visible=False)
+    estimated_aws_costs_number = gr.Number(label = "Approximate AWS Textract and/or Comprehend cost ($)", value=0, visible=False, precision=2)
+    estimated_time_taken_number = gr.Number(label = "Approximate time taken to extract text/redact (minutes)", value=0, visible=False, precision=2)
+
+    only_extract_text_radio = gr.Checkbox(value=False, label="Only extract text (no redaction)", visible=False)
 
     ###
     # UI DESIGN
@@ -174,35 +181,61 @@ with app:
     gr.Markdown(
     """# Document redaction
 
-    Redact personally identifiable information (PII) from documents (pdf, images), open text, or tabular data (xlsx/csv/parquet). Please see the [User Guide](https://github.com/seanpedrick-case/doc_redaction/blob/main/README.md) for a walkthrough on how to use the app. Below is a very brief overview.
+    Redact personally identifiable information (PII) from documents (PDF, images), open text, or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://github.com/seanpedrick-case/doc_redaction/blob/main/README.md) for a walkthrough on how to use the app. Below is a very brief overview.
     
-    To identify text in documents, the 'local' text/OCR image analysis uses spacy/tesseract, and works ok for documents with typed text. If available, choose 'AWS Textract service' to redact more complex elements e.g. signatures or handwriting. Then, choose a method for PII identification. 'Local' is quick and gives good results if you are primarily looking for a custom list of terms to redact (see Redaction settings). If available, AWS Comprehend gives better results at a small cost.
+    To identify text in documents, the 'Local' text/OCR image analysis uses spacy/tesseract, and works ok for documents with typed text. If available, choose 'AWS Textract' to redact more complex elements e.g. signatures or handwriting. Then, choose a method for PII identification. 'Local' is quick and gives good results if you are primarily looking for a custom list of terms to redact (see Redaction settings). If available, AWS Comprehend gives better results at a small cost.
     
-    After redaction, review suggested redactions on the 'Review redactions' tab. The original pdf can be uploaded here alongside a '...redaction_file.csv' to continue a previous redaction/review task. See the 'Redaction settings' tab to choose which pages to redact, the type of information to redact (e.g. people, places), or custom terms to always include/ exclude from redaction.
+    After redaction, review suggested redactions on the 'Review redactions' tab. The original pdf can be uploaded here alongside a '...review_file.csv' to continue a previous redaction/review task. See the 'Redaction settings' tab to choose which pages to redact, the type of information to redact (e.g. people, places), or custom terms to always include/ exclude from redaction.
 
     NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.""")
 
     ###
-    # REDACTION PDF/IMAGES TABL
+    # REDACTION PDF/IMAGES TABLE
     ###
     with gr.Tab("Redact PDFs/images"):
         with gr.Accordion("Redact document", open = True):
             in_doc_files = gr.File(label="Choose a document or image file (PDF, JPG, PNG)", file_count= "multiple", file_types=['.pdf', '.jpg', '.png', '.json', '.zip'], height=file_input_height)
-            # if RUN_AWS_FUNCTIONS == "1":
-            in_redaction_method = gr.Radio(label="Choose text extraction method. AWS Textract has a cost per page - $3.50 per 1,000 pages with signature detection (default), $1.50 without. Go to Redaction settings - AWS Textract options to remove signature detection.", value = default_ocr_val, choices=[text_ocr_option, tesseract_ocr_option, textract_option])
-            pii_identification_method_drop = gr.Radio(label = "Choose PII detection method. AWS Comprehend has a cost of approximately $0.01 per 10,000 characters.", value = default_pii_detector, choices=[local_pii_detector, aws_pii_detector])
-            # else:
-            #     in_redaction_method = gr.Radio(label="Choose text extraction method.", value = default_ocr_val, choices=[text_ocr_option, tesseract_ocr_option])
-            #     pii_identification_method_drop = gr.Radio(label = "Choose PII detection method.", value = default_pii_detector, choices=[local_pii_detector], visible=False)
-
-            gr.Markdown("""If you only want to redact certain pages, or certain entities (e.g. just email addresses, or a custom list of terms), please go to the redaction settings tab.""")
-            document_redact_btn = gr.Button("Redact document", variant="primary")
-            current_loop_page_number = gr.Number(value=0,precision=0, interactive=False, label = "Last redacted page in document", visible=False)
-            page_break_return = gr.Checkbox(value = False, label="Page break reached", visible=False)
+
+            text_extract_method_radio = gr.Radio(label="Choose text extraction method. AWS Textract has a cost per page - $3.50 per 1,000 pages with signature detection (default), $1.50 without. Go to Redaction settings - AWS Textract options to remove signature detection.", value = default_ocr_val, choices=[text_ocr_option, tesseract_ocr_option, textract_option])
+
+            with gr.Row(equal_height=True):                
+                pii_identification_method_drop = gr.Radio(label = "Choose PII detection method. AWS Comprehend has a cost of approximately $0.01 per 10,000 characters.", value = default_pii_detector, choices=[no_redaction_option, local_pii_detector, aws_pii_detector])
+
+            with gr.Accordion("AWS Textract signature detection (default is on)", open = False):
+                handwrite_signature_checkbox = gr.CheckboxGroup(label="AWS Textract extraction settings", choices=["Extract handwriting", "Extract signatures"], value=["Extract handwriting", "Extract signatures"])            
+
+            if SHOW_BULK_TEXTRACT_CALL_OPTIONS == "True":
+                with gr.Accordion("AWS Textract bulk document API call", open = False, visible=True):
+                    with gr.Row(equal_height=True):                    
+                        job_name_textbox = gr.Textbox(value="", label="Bulk Textract call", visible=True)
+                        send_document_to_textract_api_btn = gr.Button("Analyse document with AWS Textract", variant="primary", visible=True)
+                    with gr.Row(equal_height=True):                    
+                        check_state_of_textract_api__call_btn = gr.Button("Check state of Textract job", variant="secondary", visible=True)
+                        job_current_status = gr.Textbox(value="", label="job_current_status", visible=True)
+                    with gr.Row(equal_height=True):
+                        textract_job_output_file = gr.File(label="Textract job output files", height=file_input_height, visible=True)
+            
+            if SHOW_COSTS == "True":
+                with gr.Accordion("Estimated costs and time taken", open = False, visible=True):                        
+                        with gr.Row(equal_height=True):
+                            textract_output_found_checkbox = gr.Checkbox(value= False, label="Existing Textract output file found", interactive=False, visible=True)
+                            total_pdf_page_count = gr.Number(label = "Total page count", value=0, visible=True)
+                            estimated_aws_costs_number = gr.Number(label = "Approximate AWS Textract and/or Comprehend cost ($)", value=0.00, precision=2, visible=True)
+                            estimated_time_taken_number = gr.Number(label = "Approximate time taken to extract text/redact (minutes)", value=0, visible=True, precision=2)
+
+            gr.Markdown("""If you only want to redact certain pages, or certain entities (e.g. just email addresses, or a custom list of terms), please go to the Redaction Settings tab.""")            
+                
+            document_redact_btn = gr.Button("Extract text and redact document", variant="primary", scale = 4)
+
+            if GET_COST_CODES == "True" or ENFORCE_COST_CODES == "True":
+                with gr.Accordion("Apply cost code", open = True, visible=True):
+                    with gr.Row(equal_height=True):
+                        cost_code_dataframe = gr.Dataframe(value=pd.DataFrame(), row_count = (0, "dynamic"), label="Cost codes", type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True, show_search='search', visible=True, wrap=True, max_height=200)
+                        cost_code_choice_drop = gr.Dropdown(value="", label="Choose cost code for analysis", choices=[], allow_custom_value=True, visible=True) 
         
         with gr.Row():
             output_summary = gr.Textbox(label="Output summary", scale=1)
-            output_file = gr.File(label="Output files", scale = 2, height=file_input_height)
+            output_file = gr.File(label="Output files", scale = 2)#, height=file_input_height)
             latest_file_completed_text = gr.Number(value=0, label="Number of documents redacted", interactive=False, visible=False)
 
         with gr.Row():
@@ -221,7 +254,7 @@ with app:
 
         with gr.Accordion(label = "Review PDF redactions", open=True):
             output_review_files = gr.File(label="Upload original PDF and 'review_file' csv here to review suggested redactions", file_count='multiple', height=file_input_height)
-            upload_previous_review_file_btn = gr.Button("Review PDF and 'review file' csv provided above", variant="primary")                   
+            upload_previous_review_file_btn = gr.Button("Review PDF and 'review file' csv provided above", variant="secondary")                   
         with gr.Row():
             annotate_zoom_in = gr.Button("Zoom in", visible=False)
             annotate_zoom_out = gr.Button("Zoom out", visible=False)        
@@ -236,12 +269,10 @@ with app:
                     annotate_max_pages = gr.Number(value=1, label="Total pages", precision=0, interactive=False, scale = 2, min_width=50)
                     annotation_next_page_button = gr.Button("Next page", scale = 4)
             with gr.Column(scale=1):
-                annotation_button_apply = gr.Button("Apply revised redactions to PDF", variant="secondary") 
-                #blank_markdown_top = gr.Markdown(value="", label="")
+                annotation_button_apply = gr.Button("Apply revised redactions to PDF", variant="primary") 
 
         with gr.Row():
             with gr.Column(scale=2):
-
                 zoom_str = str(annotator_zoom_number) + '%'
 
                 annotator = image_annotator(
@@ -266,11 +297,17 @@ with app:
                     recogniser_entity_dropdown = gr.Dropdown(label="Redaction category", value="ALL", allow_custom_value=True)
                     page_entity_dropdown = gr.Dropdown(label="Page", value="ALL", allow_custom_value=True)                    
                 text_entity_dropdown = gr.Dropdown(label="Text", value="ALL", allow_custom_value=True)
-                recogniser_entity_dataframe = gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), col_count=(3,"fixed"), type="pandas", label="Search results. Click to go to page", headers=["page", "label", "text"])
+                recogniser_entity_dataframe = gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), col_count=(3,"fixed"), type="pandas", label="Search results. Click to go to page", headers=["page", "label", "text"], show_fullscreen_button=True, wrap=True)
+
+                with gr.Row(equal_height=True):
+                    exclude_selected_row_btn = gr.Button(value="Exclude specific row from redactions")
+                    exclude_selected_btn = gr.Button(value="Exclude all items in table from redactions")
                 with gr.Row(equal_height=True):
                     reset_dropdowns_btn = gr.Button(value="Reset filters")
-                    exclude_selected_btn = gr.Button(value="Exclude items in table from redactions")
+                    
                 undo_last_removal_btn = gr.Button(value="Undo last element removal")
+                update_current_page_redactions_btn = gr.Button(value="Save changes on current page to file", variant="primary")
+                selected_entity_dataframe_row = gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), col_count=3, type="pandas", visible=False, label="selected_entity_dataframe_row", headers=["page", "label", "text"], show_fullscreen_button=True, wrap=True)
 
         with gr.Row(): 
             with gr.Column(scale=2):
@@ -285,17 +322,25 @@ with app:
         with gr.Accordion("Convert review files loaded above to Adobe format, or convert from Adobe format to review file", open = False):
             convert_review_file_to_adobe_btn = gr.Button("Convert review file to Adobe comment format", variant="primary")
             adobe_review_files_out = gr.File(label="Output Adobe comment files will appear here. If converting from .xfdf file to review_file.csv, upload the original pdf with the xfdf file here then click Convert below.", file_count='multiple', file_types=['.csv', '.xfdf', '.pdf']) 
-            convert_adobe_to_review_file_btn = gr.Button("Convert Adobe .xfdf comment file to review_file.csv", variant="secondary")   
-        
+            convert_adobe_to_review_file_btn = gr.Button("Convert Adobe .xfdf comment file to review_file.csv", variant="secondary")
+
+    ###
+    # IDENTIFY DUPLICATE PAGES TAB
+    ###
+    with gr.Tab(label="Identify duplicate pages"):
+        with gr.Accordion("Identify duplicate pages to redact", open = True):            
+            in_duplicate_pages = gr.File(label="Upload multiple 'ocr_output.csv' data files from redaction jobs here to compare", file_count="multiple", height=file_input_height, file_types=['.csv'])
+            with gr.Row():
+                duplicate_threshold_value = gr.Number(value=0.9, label="Minimum similarity to be considered a duplicate (maximum = 1)", scale  =1)
+                find_duplicate_pages_btn = gr.Button(value="Identify duplicate pages", variant="primary", scale = 5)                
+
+            duplicate_pages_out = gr.File(label="Duplicate pages analysis output", file_count="multiple", height=file_input_height, file_types=['.csv'])
+
     ###
     # TEXT / TABULAR DATA TAB
     ###
     with gr.Tab(label="Open text or Excel/csv files"):
-        gr.Markdown(
-    """
-    ### Choose open text or a tabular data file (xlsx or csv) to redact.
-    """
-        )    
+        gr.Markdown("""### Choose open text or a tabular data file (xlsx or csv) to redact.""")    
         with gr.Accordion("Paste open text", open = False):
             in_text = gr.Textbox(label="Enter open text", lines=10)
         with gr.Accordion("Upload xlsx or csv files", open = True):
@@ -321,15 +366,6 @@ with app:
         data_further_details_text = gr.Textbox(label="Please give more detailed feedback about the results:", visible=False)
         data_submit_feedback_btn = gr.Button(value="Submit feedback", visible=False)
 
-    ###
-    # IDENTIFY DUPLICATE PAGES TAB
-    ###
-    with gr.Tab(label="Identify duplicate pages"):
-        with gr.Accordion("Identify duplicate pages to redact", open = True):            
-            in_duplicate_pages = gr.File(label="Upload multiple 'ocr_output.csv' data files from redaction jobs here to compare", file_count="multiple", height=file_input_height, file_types=['.csv'])
-            find_duplicate_pages_btn = gr.Button(value="Identify duplicate pages", variant="primary")                
-            duplicate_pages_out =gr.File(label="Duplicate pages analysis output", file_count="multiple", height=file_input_height, file_types=['.csv'])
-
     ###
     # SETTINGS TAB
     ###
@@ -347,9 +383,9 @@ with app:
                     in_fully_redacted_list_text = gr.Textbox(label="Fully redacted page list load status")
             with gr.Accordion("Manually modify custom allow, deny, and full page redaction lists (NOTE: you need to press Enter after modifying/adding an entry to the lists to apply them)", open = False):
                 with gr.Row():
-                    in_allow_list_state = gr.Dataframe(value=pd.DataFrame(), headers=["allow_list"], col_count=(1, "fixed"), row_count = (0, "dynamic"), label="Allow list", visible=True, type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True)
-                    in_deny_list_state = gr.Dataframe(value=pd.DataFrame(), headers=["deny_list"], col_count=(1, "fixed"), row_count = (0, "dynamic"), label="Deny list", visible=True, type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True)
-                    in_fully_redacted_list_state = gr.Dataframe(value=pd.DataFrame(), headers=["fully_redacted_pages_list"], col_count=(1, "fixed"), row_count = (0, "dynamic"), label="Fully redacted pages", visible=True, type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True, datatype='number')
+                    in_allow_list_state = gr.Dataframe(value=pd.DataFrame(), headers=["allow_list"], col_count=(1, "fixed"), row_count = (0, "dynamic"), label="Allow list", visible=True, type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True, wrap=True)
+                    in_deny_list_state = gr.Dataframe(value=pd.DataFrame(), headers=["deny_list"], col_count=(1, "fixed"), row_count = (0, "dynamic"), label="Deny list", visible=True, type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True, wrap=True)
+                    in_fully_redacted_list_state = gr.Dataframe(value=pd.DataFrame(), headers=["fully_redacted_pages_list"], col_count=(1, "fixed"), row_count = (0, "dynamic"), label="Fully redacted pages", visible=True, type="pandas", interactive=True, show_fullscreen_button=True, show_copy_button=True, datatype='number', wrap=True)
             
         with gr.Accordion("Select entity types to redact", open = True):
                 in_redact_entities = gr.Dropdown(value=chosen_redact_entities, choices=full_entity_list, multiselect=True, label="Local PII identification model (click empty space in box for full list)")
@@ -364,10 +400,9 @@ with app:
                 page_min = gr.Number(precision=0,minimum=0,maximum=9999, label="Lowest page to redact")
                 page_max = gr.Number(precision=0,minimum=0,maximum=9999, label="Highest page to redact")
 
-        with gr.Accordion("AWS Textract options", open = False):            
-            handwrite_signature_checkbox = gr.CheckboxGroup(label="AWS Textract settings", choices=["Redact all identified handwriting", "Redact all identified signatures"], value=["Redact all identified handwriting", "Redact all identified signatures"])
+        with gr.Accordion("AWS options", open = False):
             #with gr.Row():
-            in_redact_language = gr.Dropdown(value = "en", choices = ["en"], label="Redaction language (only English currently supported)", multiselect=False, visible=False)
+            in_redact_language = gr.Dropdown(value = REDACTION_LANGUAGE, choices = [REDACTION_LANGUAGE], label="Redaction language", multiselect=False, visible=False)
 
             with gr.Row():
                 aws_access_key_textbox = gr.Textbox(value='', label="AWS access key for account with permissions for AWS Textract and Comprehend", visible=True, type="password")
@@ -384,31 +419,56 @@ with app:
 
         with gr.Accordion("View all output files from this session", open = False):
             all_output_files_btn = gr.Button("Click here to view all output files", variant="secondary")
-            all_output_files = gr.File(label="All output files.", file_count='multiple', file_types=['.csv'], interactive=False)
-
+            all_output_files = gr.File(label="All files in output folder", file_count='multiple', file_types=['.csv'], interactive=False)
 
     ### UI INTERACTION ###
 
     ###
     # PDF/IMAGE REDACTION
     ###
-    in_doc_files.upload(fn=get_input_file_names, inputs=[in_doc_files], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list])
+    # Recalculate estimated costs based on changes to inputs
+    if SHOW_COSTS == 'True':
+        # Calculate costs
+        total_pdf_page_count.change(calculate_aws_costs, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox,         pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_aws_costs_number])
+        text_extract_method_radio.change(calculate_aws_costs, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox, pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_aws_costs_number])
+        pii_identification_method_drop.change(calculate_aws_costs, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox,  pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_aws_costs_number])
+        handwrite_signature_checkbox.change(calculate_aws_costs, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox,  pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_aws_costs_number])
+        textract_output_found_checkbox.change(calculate_aws_costs, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox,  pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_aws_costs_number])
+        only_extract_text_radio.change(calculate_aws_costs, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox,  pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_aws_costs_number])
+
+        # Calculate time taken
+        total_pdf_page_count.change(calculate_time_taken, inputs=[total_pdf_page_count, text_extract_method_radio,          pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_time_taken_number])
+        text_extract_method_radio.change(calculate_time_taken, inputs=[total_pdf_page_count, text_extract_method_radio, pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_time_taken_number])
+        pii_identification_method_drop.change(calculate_time_taken, inputs=[total_pdf_page_count, text_extract_method_radio,  pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_time_taken_number])
+        handwrite_signature_checkbox.change(calculate_time_taken, inputs=[total_pdf_page_count, text_extract_method_radio, pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_time_taken_number])
+        textract_output_found_checkbox.change(calculate_time_taken, inputs=[total_pdf_page_count, text_extract_method_radio, handwrite_signature_checkbox,  pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_time_taken_number])
+        only_extract_text_radio.change(calculate_time_taken, inputs=[total_pdf_page_count, text_extract_method_radio, pii_identification_method_drop, textract_output_found_checkbox, only_extract_text_radio], outputs=[estimated_time_taken_number])
+
+    # Allow user to select items from cost code dataframe for cost code
+    if SHOW_COSTS=="True" and (GET_COST_CODES == "True" or ENFORCE_COST_CODES == "True"):
+        cost_code_dataframe.select(df_select_callback_cost, inputs=[cost_code_dataframe], outputs=[cost_code_choice_drop])
+
+    in_doc_files.upload(fn=get_input_file_names, inputs=[in_doc_files], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list, total_pdf_page_count]).\
+    success(fn = prepare_image_or_pdf, inputs=[in_doc_files, text_extract_method_radio, latest_file_completed_text, output_summary, first_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox, input_folder_textbox, prepare_images_bool_false], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes, textract_output_found_checkbox, all_img_details_state]).\
+    success(fn=check_for_existing_textract_file, inputs=[doc_file_name_no_extension_textbox, output_folder_textbox], outputs=[textract_output_found_checkbox])
 
     # Run redaction function
-    document_redact_btn.click(fn = reset_state_vars, outputs=[pdf_doc_state, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, textract_metadata_textbox, annotator, output_file_list_state, log_files_output_list_state, recogniser_entity_dataframe, recogniser_entity_dataframe_base, document_cropboxes]).\
-        success(fn = choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, in_redaction_method, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_text, output_summary, output_file_list_state, log_files_output_list_state, first_loop_state, page_min, page_max, estimated_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_state, output_folder_textbox, document_cropboxes, page_sizes],
-                    outputs=[output_summary, output_file, output_file_list_state, latest_file_completed_text, log_files_output, log_files_output_list_state, estimated_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_state, page_sizes, document_cropboxes], api_name="redact_doc").\
-                    success(fn=update_annotator, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown])
-    
+    document_redact_btn.click(fn = reset_state_vars, outputs=[all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, textract_metadata_textbox, annotator, output_file_list_state, log_files_output_list_state, recogniser_entity_dataframe, recogniser_entity_dataframe_base, pdf_doc_state, duplication_file_path_outputs_list_state, output_summary]).\
+        success(fn= enforce_cost_codes, inputs=[enforce_cost_code_textbox, cost_code_choice_drop]).\
+        success(fn= choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, text_extract_method_radio, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_text, output_summary, output_file_list_state, log_files_output_list_state, first_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_state, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number],
+                    outputs=[output_summary, output_file, output_file_list_state, latest_file_completed_text, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_state, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number], api_name="redact_doc").\
+                    success(fn=update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state])
+
     # If the app has completed a batch of pages, it will rerun the redaction process until the end of all pages in the document
-    current_loop_page_number.change(fn = choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, in_redaction_method, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_text, output_summary, output_file_list_state, log_files_output_list_state, second_loop_state, page_min, page_max, estimated_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_state, output_folder_textbox, document_cropboxes, page_sizes],
-                    outputs=[output_summary, output_file, output_file_list_state, latest_file_completed_text, log_files_output, log_files_output_list_state, estimated_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_state, page_sizes, document_cropboxes]).\
-                    success(fn=update_annotator, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown])
-    
+    current_loop_page_number.change(fn = choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, text_extract_method_radio, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_text, output_summary, output_file_list_state, log_files_output_list_state, second_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_state, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number],
+                    outputs=[output_summary, output_file, output_file_list_state, latest_file_completed_text, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_state, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number]).\
+                    success(fn=update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state])
+        
     # If a file has been completed, the function will continue onto the next document
-    latest_file_completed_text.change(fn = choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, in_redaction_method, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_text, output_summary, output_file_list_state, log_files_output_list_state, second_loop_state, page_min, page_max, estimated_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_state, output_folder_textbox, document_cropboxes, page_sizes],
-                    outputs=[output_summary, output_file, output_file_list_state, latest_file_completed_text, log_files_output, log_files_output_list_state, estimated_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_state, page_sizes, document_cropboxes]).\
-                    success(fn=update_annotator, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
+    latest_file_completed_text.change(fn = choose_and_run_redactor, inputs=[in_doc_files, prepared_pdf_state, images_pdf_state, in_redact_language, in_redact_entities, in_redact_comprehend_entities, text_extract_method_radio, in_allow_list_state, in_deny_list_state, in_fully_redacted_list_state, latest_file_completed_text, output_summary, output_file_list_state, log_files_output_list_state, second_loop_state, page_min, page_max, actual_time_taken_number, handwrite_signature_checkbox, textract_metadata_textbox, all_image_annotations_state, all_line_level_ocr_results_df_state, all_decision_process_table_state, pdf_doc_state, current_loop_page_number, page_break_return, pii_identification_method_drop, comprehend_query_number, max_fuzzy_spelling_mistakes_num, match_fuzzy_whole_phrase_bool, aws_access_key_textbox, aws_secret_key_textbox, annotate_max_pages, review_file_state, output_folder_textbox, document_cropboxes, page_sizes, textract_output_found_checkbox, only_extract_text_radio, duplication_file_path_outputs_list_state, latest_review_file_path, input_folder_textbox, textract_query_number],
+                    outputs=[output_summary, output_file, output_file_list_state, latest_file_completed_text, log_files_output, log_files_output_list_state, actual_time_taken_number, textract_metadata_textbox, pdf_doc_state, all_image_annotations_state, current_loop_page_number, page_break_return, all_line_level_ocr_results_df_state, all_decision_process_table_state, comprehend_query_number, output_review_files, annotate_max_pages, annotate_max_pages_bottom, prepared_pdf_state, images_pdf_state, review_file_state, page_sizes, duplication_file_path_outputs_list_state, in_duplicate_pages, latest_review_file_path, textract_query_number]).\
+                    success(fn=update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, page_min, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs=[annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state]).\
+                    success(fn=check_for_existing_textract_file, inputs=[doc_file_name_no_extension_textbox, output_folder_textbox], outputs=[textract_output_found_checkbox]).\
                     success(fn=reveal_feedback_buttons, outputs=[pdf_feedback_radio, pdf_further_details_text, pdf_submit_feedback_btn, pdf_feedback_title])
     
     ###
@@ -417,92 +477,75 @@ with app:
 
     # Upload previous files for modifying redactions
     upload_previous_review_file_btn.click(fn=reset_review_vars, inputs=None, outputs=[recogniser_entity_dataframe, recogniser_entity_dataframe_base]).\
-        success(fn=get_input_file_names, inputs=[output_review_files], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list]).\
-        success(fn = prepare_image_or_pdf, inputs=[output_review_files, in_redaction_method, latest_file_completed_text, output_summary, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes], api_name="prepare_doc").\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown])
-
-    # Page controls at top
-    annotate_current_page.submit(
-        modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
+        success(fn=get_input_file_names, inputs=[output_review_files], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list, total_pdf_page_count]).\
+        success(fn = prepare_image_or_pdf, inputs=[output_review_files, text_extract_method_radio, latest_file_completed_text, output_summary, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox, input_folder_textbox, prepare_images_bool_false], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes, textract_output_found_checkbox, all_img_details_state], api_name="prepare_doc").\
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state])
+
+    # Page number controls
+    annotate_current_page.change(update_all_page_annotation_object_based_on_previous_page, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state]).\
+        success(apply_redactions_to_review_df_and_files, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output, review_file_state])
     
-    annotation_last_page_button.click(fn=decrease_page, inputs=[annotate_current_page], outputs=[annotate_current_page, annotate_current_page_bottom]).\
-        success(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])    
-
-    annotation_next_page_button.click(fn=increase_page, inputs=[annotate_current_page, all_image_annotations_state], outputs=[annotate_current_page, annotate_current_page_bottom]).\
-        success(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
-    
-    # Zoom in and out on annotator
-    annotate_zoom_in.click(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_zoom, inputs=[annotator_zoom_number, annotate_current_page, zoom_true_bool], outputs=[annotator_zoom_number, annotate_current_page])
-        
-    annotate_zoom_out.click(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_zoom, inputs=[annotator_zoom_number, annotate_current_page, zoom_false_bool], outputs=[annotator_zoom_number, annotate_current_page])
-    
-    annotator_zoom_number.change(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base])
-
-    clear_all_redactions_on_page_btn.click(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, clear_all_page_redactions], outputs = [all_image_annotations_state, annotate_previous_page]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base])
+    annotation_last_page_button.click(fn=decrease_page, inputs=[annotate_current_page], outputs=[annotate_current_page, annotate_current_page])
+    annotation_next_page_button.click(fn=increase_page, inputs=[annotate_current_page, all_image_annotations_state], outputs=[annotate_current_page, annotate_current_page])        
 
-    annotation_button_apply.click(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output], scroll_to_output=True)
+    annotation_last_page_button_bottom.click(fn=decrease_page, inputs=[annotate_current_page], outputs=[annotate_current_page_bottom, annotate_current_page_bottom])    
+    annotation_next_page_button_bottom.click(fn=increase_page, inputs=[annotate_current_page, all_image_annotations_state], outputs=[annotate_current_page_bottom, annotate_current_page_bottom])
 
-    # Page controls at bottom
-    annotate_current_page_bottom.submit(
-        modify_existing_page_redactions, inputs = [annotator, annotate_current_page_bottom, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
+    annotate_current_page_bottom.submit(update_other_annotator_number_from_current, inputs=[annotate_current_page_bottom], outputs=[annotate_current_page])
 
-    annotation_last_page_button_bottom.click(fn=decrease_page, inputs=[annotate_current_page], outputs=[annotate_current_page, annotate_current_page_bottom]).\
-        success(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
-    
-    annotation_next_page_button_bottom.click(fn=increase_page, inputs=[annotate_current_page, all_image_annotations_state], outputs=[annotate_current_page, annotate_current_page_bottom]).\
-        success(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
+    # Apply page redactions
+    annotation_button_apply.click(apply_redactions_to_review_df_and_files, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output, review_file_state], scroll_to_output=True)
     
     # Review table controls
     recogniser_entity_dropdown.select(update_entities_df_recogniser_entities, inputs=[recogniser_entity_dropdown, recogniser_entity_dataframe_base, page_entity_dropdown, text_entity_dropdown], outputs=[recogniser_entity_dataframe, text_entity_dropdown, page_entity_dropdown])
     page_entity_dropdown.select(update_entities_df_page, inputs=[page_entity_dropdown, recogniser_entity_dataframe_base, recogniser_entity_dropdown, text_entity_dropdown], outputs=[recogniser_entity_dataframe, recogniser_entity_dropdown, text_entity_dropdown])
     text_entity_dropdown.select(update_entities_df_text, inputs=[text_entity_dropdown, recogniser_entity_dataframe_base, recogniser_entity_dropdown, page_entity_dropdown], outputs=[recogniser_entity_dataframe, recogniser_entity_dropdown, page_entity_dropdown])
 
-    recogniser_entity_dataframe.select(df_select_callback, inputs=[recogniser_entity_dataframe], outputs=[annotate_current_page]).\
-    success(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
+    recogniser_entity_dataframe.select(df_select_callback, inputs=[recogniser_entity_dataframe], outputs=[annotate_current_page, selected_entity_dataframe_row])#.\
+        #success(update_selected_review_df_row_colour, inputs=[selected_entity_dataframe_row, review_file_state], outputs=[review_file_state]).\
+        #success(update_annotator_page_from_review_df, inputs=[review_file_state, images_pdf_state, page_sizes, annotate_current_page, annotate_previous_page, all_image_annotations_state, annotator], outputs=[annotator, all_image_annotations_state])
     
-    reset_dropdowns_btn.click(reset_dropdowns, outputs=[recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown])
     
+    reset_dropdowns_btn.click(reset_dropdowns, inputs=[recogniser_entity_dataframe_base], outputs=[recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown]).\
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state])
+    
+    # Exclude current selection from annotator and outputs
+    # Exclude only row
+    exclude_selected_row_btn.click(exclude_selected_items_from_redaction, inputs=[review_file_state, selected_entity_dataframe_row, images_pdf_state, page_sizes, all_image_annotations_state, recogniser_entity_dataframe_base], outputs=[review_file_state, all_image_annotations_state, recogniser_entity_dataframe_base, backup_review_state, backup_image_annotations_state, backup_recogniser_entity_dataframe_base]).\
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state]).\
+        success(apply_redactions_to_review_df_and_files, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output, review_file_state]).\
+        success(update_all_entity_df_dropdowns, inputs=[recogniser_entity_dataframe_base, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown], outputs=[recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown])
+    
+    # Exclude everything visible in table
     exclude_selected_btn.click(exclude_selected_items_from_redaction, inputs=[review_file_state, recogniser_entity_dataframe, images_pdf_state, page_sizes, all_image_annotations_state, recogniser_entity_dataframe_base], outputs=[review_file_state, all_image_annotations_state, recogniser_entity_dataframe_base, backup_review_state, backup_image_annotations_state, backup_recogniser_entity_dataframe_base]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown]).\
-        success(apply_redactions, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output])
-    # success(modify_existing_page_redactions, inputs = [annotator, annotate_current_page, annotate_previous_page, all_image_annotations_state, recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown, recogniser_entity_dataframe_base, review_file_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state]).\
+        success(apply_redactions_to_review_df_and_files, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output, review_file_state]).\
+        success(update_all_entity_df_dropdowns, inputs=[recogniser_entity_dataframe_base, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown], outputs=[recogniser_entity_dropdown, text_entity_dropdown, page_entity_dropdown])
 
     undo_last_removal_btn.click(undo_last_removal, inputs=[backup_review_state, backup_image_annotations_state, backup_recogniser_entity_dataframe_base], outputs=[review_file_state, all_image_annotations_state, recogniser_entity_dataframe_base]).\
-        success(update_annotator, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown])
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state]).\
+        success(apply_redactions_to_review_df_and_files, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output, review_file_state])
+    
+    update_current_page_redactions_btn.click(update_all_page_annotation_object_based_on_previous_page, inputs = [annotator, annotate_current_page, annotate_current_page, all_image_annotations_state, page_sizes], outputs = [all_image_annotations_state, annotate_previous_page, annotate_current_page_bottom]).\
+        success(update_annotator_object_and_filter_df, inputs=[all_image_annotations_state, annotate_current_page, recogniser_entity_dropdown, page_entity_dropdown, text_entity_dropdown, recogniser_entity_dataframe_base, annotator_zoom_number, review_file_state, page_sizes, doc_full_file_name_textbox, input_folder_textbox], outputs = [annotator, annotate_current_page, annotate_current_page_bottom, annotate_previous_page, recogniser_entity_dropdown, recogniser_entity_dataframe, recogniser_entity_dataframe_base, text_entity_dropdown, page_entity_dropdown, page_sizes, all_image_annotations_state]).\
+        success(apply_redactions_to_review_df_and_files, inputs=[annotator, doc_full_file_name_textbox, pdf_doc_state, all_image_annotations_state, annotate_current_page, review_file_state, output_folder_textbox, do_not_save_pdf_state, page_sizes], outputs=[pdf_doc_state, all_image_annotations_state, output_review_files, log_files_output, review_file_state])
     
     # Convert review file to xfdf Adobe format
-    convert_review_file_to_adobe_btn.click(fn=get_input_file_names, inputs=[output_review_files], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list]).\
-        success(fn = prepare_image_or_pdf, inputs=[output_review_files, in_redaction_method, latest_file_completed_text, output_summary, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes]).\
+    convert_review_file_to_adobe_btn.click(fn=get_input_file_names, inputs=[output_review_files], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list, total_pdf_page_count]).\
+        success(fn = prepare_image_or_pdf, inputs=[output_review_files, text_extract_method_radio, latest_file_completed_text, output_summary, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox, input_folder_textbox, prepare_images_bool_false], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes, textract_output_found_checkbox, all_img_details_state]).\
         success(convert_df_to_xfdf, inputs=[output_review_files, pdf_doc_state, images_pdf_state, output_folder_textbox, document_cropboxes, page_sizes], outputs=[adobe_review_files_out])
     
     # Convert xfdf Adobe file back to review_file.csv
-    convert_adobe_to_review_file_btn.click(fn=get_input_file_names, inputs=[adobe_review_files_out], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list]).\
-        success(fn = prepare_image_or_pdf, inputs=[adobe_review_files_out, in_redaction_method, latest_file_completed_text, output_summary, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes]).\
+    convert_adobe_to_review_file_btn.click(fn=get_input_file_names, inputs=[adobe_review_files_out], outputs=[doc_file_name_no_extension_textbox, doc_file_name_with_extension_textbox, doc_full_file_name_textbox, doc_file_name_textbox_list, total_pdf_page_count]).\
+        success(fn = prepare_image_or_pdf, inputs=[adobe_review_files_out, text_extract_method_radio, latest_file_completed_text, output_summary, second_loop_state, annotate_max_pages, all_image_annotations_state, prepare_for_review_bool, in_fully_redacted_list_state, output_folder_textbox, input_folder_textbox, prepare_images_bool_false], outputs=[output_summary, prepared_pdf_state, images_pdf_state, annotate_max_pages, annotate_max_pages_bottom, pdf_doc_state, all_image_annotations_state, review_file_state, document_cropboxes, page_sizes, textract_output_found_checkbox, all_img_details_state]).\
         success(fn=convert_xfdf_to_dataframe, inputs=[adobe_review_files_out, pdf_doc_state, images_pdf_state, output_folder_textbox], outputs=[output_review_files], scroll_to_output=True)
     
     ###
     # TABULAR DATA REDACTION
     ###
     in_data_files.upload(fn=put_columns_in_df, inputs=[in_data_files], outputs=[in_colnames, in_excel_sheets]).\
-                  success(fn=get_input_file_names, inputs=[in_data_files], outputs=[data_file_name_no_extension_textbox, data_file_name_with_extension_textbox, data_full_file_name_textbox, data_file_name_textbox_list])
+                  success(fn=get_input_file_names, inputs=[in_data_files], outputs=[data_file_name_no_extension_textbox, data_file_name_with_extension_textbox, data_full_file_name_textbox, data_file_name_textbox_list, total_pdf_page_count])
 
     tabular_data_redact_btn.click(fn=anonymise_data_files, inputs=[in_data_files, in_text, anon_strat, in_colnames, in_redact_language, in_redact_entities, in_allow_list_state, text_tabular_files_done, text_output_summary, text_output_file_list_state, log_files_output_list_state, in_excel_sheets, first_loop_state, output_folder_textbox, in_deny_list_state, max_fuzzy_spelling_mistakes_num, pii_identification_method_drop_tabular, in_redact_comprehend_entities, comprehend_query_number, aws_access_key_textbox, aws_secret_key_textbox], outputs=[text_output_summary, text_output_file, text_output_file_list_state, text_tabular_files_done, log_files_output, log_files_output_list_state], api_name="redact_data")
 
@@ -513,7 +556,7 @@ with app:
     ###
     # IDENTIFY DUPLICATE PAGES
     ###
-    find_duplicate_pages_btn.click(fn=identify_similar_pages, inputs=[in_duplicate_pages], outputs=[duplicate_pages_df, duplicate_pages_out])
+    find_duplicate_pages_btn.click(fn=identify_similar_pages, inputs=[in_duplicate_pages, duplicate_threshold_value, output_folder_textbox], outputs=[duplicate_pages_df, duplicate_pages_out])
 
     ###
     # SETTINGS PAGE INPUT / OUTPUT
@@ -539,42 +582,55 @@ with app:
     ###
 
     # Get connection details on app load
-    app.load(get_connection_params, inputs=[output_folder_textbox], outputs=[session_hash_state, output_folder_textbox, session_hash_textbox])
+    app.load(get_connection_params, inputs=[output_folder_textbox, input_folder_textbox], outputs=[session_hash_state, output_folder_textbox, session_hash_textbox, input_folder_textbox])
 
-    # If relevant environment variable is set, load in the default allow list file from S3
-    if GET_DEFAULT_ALLOW_LIST == "True" and DEFAULT_ALLOW_LIST_PATH:
-        print("Loading allow list from default_allow_list_output_folder_location:", default_allow_list_loc)
-        if not os.path.exists(default_allow_list_loc):
+    # If relevant environment variable is set, load in the default allow list file from S3 or locally
+    if GET_DEFAULT_ALLOW_LIST == "True" and ALLOW_LIST_PATH:
+        print("Loading allow list from default_allow_list_output_path location:", ALLOW_LIST_PATH)
+        if not os.path.exists(ALLOW_LIST_PATH):
             app.load(download_file_from_s3, inputs=[s3_default_bucket, s3_default_allow_list_file, default_allow_list_output_folder_location]).\
             success(load_in_default_allow_list, inputs = [default_allow_list_output_folder_location], outputs=[in_allow_list])
         else:
             app.load(load_in_default_allow_list, inputs = [default_allow_list_output_folder_location], outputs=[in_allow_list])
 
+    # If relevant environment variable is set, load in the default cost code file from S3 or locally
+    if GET_COST_CODES == "True" and COST_CODES_PATH:
+        print("Loading cost codes from default_cost_codes_path location:", COST_CODES_PATH)
+        if not os.path.exists(COST_CODES_PATH):
+            app.load(download_file_from_s3, inputs=[s3_default_bucket, s3_default_cost_codes_file, default_cost_codes_output_folder_location]).\
+            success(load_in_default_cost_codes, inputs = [default_cost_codes_output_folder_location], outputs=[cost_code_dataframe, cost_code_choice_drop])
+        else:
+            app.load(load_in_default_cost_codes, inputs = [default_cost_codes_output_folder_location], outputs=[cost_code_dataframe, cost_code_choice_drop])
+
     # Log usernames and times of access to file (to know who is using the app when running on AWS)
     access_callback = CSVLogger_custom(dataset_file_name=log_file_name)
-    access_callback.setup([session_hash_textbox], access_logs_folder)
+    access_callback.setup([session_hash_textbox], ACCESS_LOGS_FOLDER)
     session_hash_textbox.change(lambda *args: access_callback.flag(list(args)), [session_hash_textbox], None, preprocess=False).\
     success(fn = upload_file_to_s3, inputs=[access_logs_state, access_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
     # User submitted feedback for pdf redactions
     pdf_callback = CSVLogger_custom(dataset_file_name=log_file_name)
-    pdf_callback.setup([pdf_feedback_radio, pdf_further_details_text, doc_file_name_no_extension_textbox], feedback_logs_folder)
+    pdf_callback.setup([pdf_feedback_radio, pdf_further_details_text, doc_file_name_no_extension_textbox], FEEDBACK_LOGS_FOLDER)
     pdf_submit_feedback_btn.click(lambda *args: pdf_callback.flag(list(args)), [pdf_feedback_radio, pdf_further_details_text, doc_file_name_no_extension_textbox], None, preprocess=False).\
     success(fn = upload_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[pdf_further_details_text])
 
     # User submitted feedback for data redactions
     data_callback = CSVLogger_custom(dataset_file_name=log_file_name)
-    data_callback.setup([data_feedback_radio, data_further_details_text, data_full_file_name_textbox], feedback_logs_folder)
+    data_callback.setup([data_feedback_radio, data_further_details_text, data_full_file_name_textbox], FEEDBACK_LOGS_FOLDER)
     data_submit_feedback_btn.click(lambda *args: data_callback.flag(list(args)), [data_feedback_radio, data_further_details_text, data_full_file_name_textbox], None, preprocess=False).\
     success(fn = upload_file_to_s3, inputs=[feedback_logs_state, feedback_s3_logs_loc_state], outputs=[data_further_details_text])
 
     # Log processing time/token usage when making a query
-    usage_callback = CSVLogger_custom(dataset_file_name=log_file_name)
-    usage_callback.setup([session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, estimated_time_taken_number, textract_metadata_textbox, pii_identification_method_drop, comprehend_query_number], usage_logs_folder)
-    latest_file_completed_text.change(lambda *args: usage_callback.flag(list(args)), [session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, estimated_time_taken_number, textract_metadata_textbox, pii_identification_method_drop, comprehend_query_number], None, preprocess=False).\
-    success(fn = upload_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
-
+    usage_callback = CSVLogger_custom(dataset_file_name=log_file_name)    
 
+    if DISPLAY_FILE_NAMES_IN_LOGS == 'True':
+        usage_callback.setup([session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop], USAGE_LOGS_FOLDER)
+        latest_file_completed_text.change(lambda *args: usage_callback.flag(list(args)), [session_hash_textbox, doc_file_name_no_extension_textbox, data_full_file_name_textbox, total_pdf_page_count, actual_time_taken_number, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop], None, preprocess=False).\
+        success(fn = upload_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
+    else:
+        usage_callback.setup([session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, data_full_file_name_textbox, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop], USAGE_LOGS_FOLDER)
+        latest_file_completed_text.change(lambda *args: usage_callback.flag(list(args)), [session_hash_textbox, blank_doc_file_name_no_extension_textbox_for_logs, data_full_file_name_textbox, actual_time_taken_number, total_pdf_page_count, textract_query_number, pii_identification_method_drop, comprehend_query_number, cost_code_choice_drop], None, preprocess=False).\
+        success(fn = upload_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
 
 if __name__ == "__main__":
 
@@ -590,7 +646,7 @@ if __name__ == "__main__":
 
         main(first_loop_state, latest_file_completed=0, output_summary="", output_file_list=None, 
          log_files_list=None, estimated_time=0, textract_metadata="", comprehend_query_num=0, 
-         current_loop_page=0, page_break=False, pdf_doc_state = [], all_image_annotations = [], all_line_level_ocr_results = pd.DataFrame(), all_decision_process_table = pd.DataFrame(),chosen_comprehend_entities = chosen_comprehend_entities, chosen_redact_entities = chosen_redact_entities, handwrite_signature_checkbox = ["Redact all identified handwriting", "Redact all identified signatures"])
+         current_loop_page=0, page_break=False, pdf_doc_state = [], all_image_annotations = [], all_line_level_ocr_results = pd.DataFrame(), all_decision_process_table = pd.DataFrame(),chosen_comprehend_entities = chosen_comprehend_entities, chosen_redact_entities = chosen_redact_entities, handwrite_signature_checkbox = ["Extract handwriting", "Extract signatures"])
 
 # AWS options - placeholder for possibility of storing data on s3 and retrieving it in app
 # with gr.Tab(label="Advanced options"):

requirements.txt

@@ -2,17 +2,17 @@ pdfminer.six==20240706
 pdf2image==1.17.0
 pymupdf==1.25.3
 opencv-python==4.10.0.84
-presidio_analyzer==2.2.357
-presidio_anonymizer==2.2.357
-presidio-image-redactor==0.0.55
+presidio_analyzer==2.2.358
+presidio_anonymizer==2.2.358
+presidio-image-redactor==0.0.56
 pikepdf==9.5.2
 pandas==2.2.3
-nltk==3.9.1
+#nltk==3.9.1 # Not required
 scikit-learn==1.6.1
 spacy==3.8.4
 en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0.tar.gz
 #en_core_web_sm @ https://github.com/explosion/spacy-#models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0.tar.gz
-gradio==5.22.0
+gradio==5.23.3
 boto3==1.37.17
 pyarrow==19.0.1
 openpyxl==3.1.5

tools/auth.py

@@ -4,7 +4,7 @@ import boto3
 import hmac
 import hashlib
 import base64
-from tools.config import client_id, client_secret, user_pool_id
+from tools.config import AWS_CLIENT_ID, AWS_CLIENT_SECRET, AWS_USER_POOL_ID, AWS_REGION
 
 def calculate_secret_hash(client_id:str, client_secret:str, username:str):
     message = username + client_id
@@ -16,7 +16,7 @@ def calculate_secret_hash(client_id:str, client_secret:str, username:str):
     secret_hash = base64.b64encode(dig).decode()
     return secret_hash
 
-def authenticate_user(username:str, password:str, user_pool_id:str=user_pool_id, client_id:str=client_id, client_secret:str=client_secret):
+def authenticate_user(username:str, password:str, user_pool_id:str=AWS_USER_POOL_ID, client_id:str=AWS_CLIENT_ID, client_secret:str=AWS_CLIENT_SECRET):
     """Authenticates a user against an AWS Cognito user pool.
 
     Args:
@@ -30,7 +30,7 @@ def authenticate_user(username:str, password:str, user_pool_id:str=user_pool_id,
         bool: True if the user is authenticated, False otherwise.
     """
 
-    client = boto3.client('cognito-idp')  # Cognito Identity Provider client
+    client = boto3.client('cognito-idp', region_name=AWS_REGION)  # Cognito Identity Provider client
 
     # Compute the secret hash
     secret_hash = calculate_secret_hash(client_id, client_secret, username)

tools/aws_functions.py

@@ -4,13 +4,8 @@ import boto3
 import tempfile
 import os
 from tools.config import AWS_REGION, RUN_AWS_FUNCTIONS, DOCUMENT_REDACTION_BUCKET
-
-
 PandasDataFrame = Type[pd.DataFrame]
 
-# Get AWS credentials
-bucket_name = DOCUMENT_REDACTION_BUCKET
-
 def get_assumed_role_info():
     sts_endpoint = 'https://sts.' + AWS_REGION + '.amazonaws.com'
     sts = boto3.client('sts', region_name=AWS_REGION, endpoint_url=sts_endpoint)
@@ -26,7 +21,7 @@ def get_assumed_role_info():
 
 if RUN_AWS_FUNCTIONS == "1":
     try:        
-        session = boto3.Session()   
+        session = boto3.Session(region_name=AWS_REGION)   
             
     except Exception as e:
         print("Could not start boto3 session:", e)
@@ -34,6 +29,7 @@ if RUN_AWS_FUNCTIONS == "1":
     try:
         assumed_role_arn, assumed_role_name = get_assumed_role_info()
 
+        print("Successfully assumed ARN role")
         print("Assumed Role ARN:", assumed_role_arn)
         print("Assumed Role Name:", assumed_role_name)
 
@@ -43,15 +39,15 @@ if RUN_AWS_FUNCTIONS == "1":
 # Download direct from S3 - requires login credentials
 def download_file_from_s3(bucket_name, key, local_file_path_and_name):
 
-    s3 = boto3.client('s3')
+    s3 = boto3.client('s3', region_name=AWS_REGION)
     s3.download_file(bucket_name, key, local_file_path_and_name)
     print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path_and_name}")
                          
-def download_folder_from_s3(bucket_name, s3_folder, local_folder):
+def download_folder_from_s3(bucket_name:str, s3_folder:str, local_folder:str):
     """
     Download all files from an S3 folder to a local folder.
     """
-    s3 = boto3.client('s3')
+    s3 = boto3.client('s3', region_name=AWS_REGION)
 
     # List objects in the specified S3 folder
     response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
@@ -72,11 +68,11 @@ def download_folder_from_s3(bucket_name, s3_folder, local_folder):
         except Exception as e:
             print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
 
-def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
+def download_files_from_s3(bucket_name:str, s3_folder:str, local_folder:str, filenames:List[str]):
     """
     Download specific files from an S3 folder to a local folder.
     """
-    s3 = boto3.client('s3')
+    s3 = boto3.client('s3', region_name=AWS_REGION)
 
     print("Trying to download file: ", filenames)
 
@@ -105,7 +101,7 @@ def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
         except Exception as e:
             print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
 
-def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_name):
+def load_data_from_aws(in_aws_keyword_file, aws_password:str="", bucket_name:str=DOCUMENT_REDACTION_BUCKET):
 
     temp_dir = tempfile.mkdtemp()
     local_address_stub = temp_dir + '/doc-redaction/'
@@ -156,7 +152,7 @@ def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_
 
     return files, out_message
 
-def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=bucket_name):
+def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=DOCUMENT_REDACTION_BUCKET):
     """
     Uploads a file from local machine to Amazon S3.
 
@@ -170,7 +166,7 @@ def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=buck
     """
     final_out_message = []
 
-    s3_client = boto3.client('s3')
+    s3_client = boto3.client('s3', region_name=AWS_REGION)
 
     if isinstance(local_file_paths, str):
         local_file_paths = [local_file_paths]

tools/aws_textract.py

@@ -7,7 +7,7 @@ from collections import defaultdict
 import pikepdf
 import time
 from tools.custom_image_analyser_engine import OCRResult, CustomImageRecognizerResult
-from tools.config import AWS_ACCESS_KEY, AWS_SECRET_KEY
+from tools.config import AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION
 
 def extract_textract_metadata(response:object):
     """Extracts metadata from an AWS Textract response."""
@@ -25,7 +25,7 @@ def extract_textract_metadata(response:object):
         #'NumberOfPages': number_of_pages
     })
 
-def analyse_page_with_textract(pdf_page_bytes:object, page_no:int, client:str="", handwrite_signature_checkbox:List[str]=["Redact all identified handwriting", "Redact all identified signatures"]):
+def analyse_page_with_textract(pdf_page_bytes:object, page_no:int, client:str="", handwrite_signature_checkbox:List[str]=["Extract handwriting", "Redact all identified signatures"]):
     '''
     Analyse page with AWS Textract
     '''
@@ -34,9 +34,9 @@ def analyse_page_with_textract(pdf_page_bytes:object, page_no:int, client:str=""
             if AWS_ACCESS_KEY and AWS_SECRET_KEY:
                 client = boto3.client('textract', 
                 aws_access_key_id=AWS_ACCESS_KEY, 
-                aws_secret_access_key=AWS_SECRET_KEY)
+                aws_secret_access_key=AWS_SECRET_KEY, region_name=AWS_REGION)
             else:
-                client = boto3.client('textract')
+                client = boto3.client('textract', region_name=AWS_REGION)
         except:
             print("Cannot connect to AWS Textract")
             return [], ""  # Return an empty list and an empty string
@@ -75,8 +75,12 @@ def analyse_page_with_textract(pdf_page_bytes:object, page_no:int, client:str=""
         'data': response
     }
 
+    #print("response:", response)
+
     request_metadata = extract_textract_metadata(response)  # Metadata comes out as a string
 
+    #print("request_metadata:", request_metadata)
+
     # Return a list containing the wrapped response and the metadata
     return wrapped_response, request_metadata  # Return as a list to match the desired structure
 
@@ -125,6 +129,8 @@ def json_to_ocrresult(json_data:dict, page_width:float, page_height:float, page_
     # Find the specific page data
     page_json_data = json_data #next((page for page in json_data["pages"] if page["page_no"] == page_no), None)
 
+    #print("page_json_data:", page_json_data)
+
     if "Blocks" in page_json_data:
         # Access the data for the specific page
         text_blocks = page_json_data["Blocks"]  # Access the Blocks within the page data
@@ -271,8 +277,7 @@ def json_to_ocrresult(json_data:dict, page_width:float, page_height:float, page_
 
 def load_and_convert_textract_json(textract_json_file_path:str, log_files_output_paths:str):
     """
-    Loads Textract JSON from a file, detects if conversion is needed,
-    and converts if necessary.
+    Loads Textract JSON from a file, detects if conversion is needed, and converts if necessary.
     """
     
     if not os.path.exists(textract_json_file_path):
@@ -295,15 +300,16 @@ def load_and_convert_textract_json(textract_json_file_path:str, log_files_output
 
     # Check if conversion is needed
     if "pages" in textract_data:
-        print("JSON already in the new format. No changes needed.")
+        print("JSON already in the correct format for app. No changes needed.")
         return textract_data, False, log_files_output_paths  # No conversion required
 
     if "Blocks" in textract_data:
         print("Need to convert Textract JSON to app format.")
         try:
-            from tools.aws_textract import restructure_textract_output
+            
             textract_data = restructure_textract_output(textract_data)
             return textract_data, False, log_files_output_paths  # Successfully converted
+        
         except Exception as e:
             print("Failed to convert JSON data to app format due to:", e)
             return {}, True, log_files_output_paths  # Conversion failed
@@ -312,33 +318,35 @@ def load_and_convert_textract_json(textract_json_file_path:str, log_files_output
         print("textract data:", textract_data)
         return {}, True, log_files_output_paths  # Return empty data if JSON is not recognized
 
-# Load Textract JSON output (assuming it's stored in a variable called `textract_output`)
-def restructure_textract_output(textract_output:object):
-    '''
-    Reorganise textract output that comes from the bulk textract analysis option on AWS to format that works in this app.
-    '''
-    pages_dict = defaultdict(lambda: {"page_no": None, "data": {"Blocks": []}})
+def restructure_textract_output(textract_output: dict):
+    """
+    Reorganise Textract output from the bulk Textract analysis option on AWS 
+    into a format that works in this redaction app, reducing size.
+    """
+    pages_dict = {}
 
-    # Extract number of pages from DocumentMetadata
-    total_pages = textract_output.get("DocumentMetadata", {}).get("Pages", 1)
+    # Extract total pages from DocumentMetadata
+    document_metadata = textract_output.get("DocumentMetadata", {})
 
     for block in textract_output.get("Blocks", []):
-        page_no = block.get("Page", 1)  # Default to 1 if not present
-        
-        # Ensure page metadata is only set once
-        if pages_dict[page_no]["page_no"] is None:
-            pages_dict[page_no]["page_no"] = str(page_no)
+        page_no = block.get("Page", 1)  # Default to 1 if missing
 
-        # Add block to corresponding page
-        pages_dict[page_no]["data"]["Blocks"].append(block)
+        # Initialize page structure if not already present
+        if page_no not in pages_dict:
+            pages_dict[page_no] = {"page_no": str(page_no), "data": {"Blocks": []}}
 
-    # Convert dictionary to sorted list of pages
+        # Keep only essential fields to reduce size
+        filtered_block = {
+            key: block[key] for key in ["BlockType", "Confidence", "Text", "Geometry", "Page", "Id", "Relationships"]
+            if key in block
+        }
+        
+        pages_dict[page_no]["data"]["Blocks"].append(filtered_block)
+
+    # Convert pages dictionary to a sorted list
     structured_output = {
+        "DocumentMetadata": document_metadata,  # Store metadata separately
         "pages": [pages_dict[page] for page in sorted(pages_dict.keys())]
     }
 
-    # Add DocumentMetadata to the first page's data (optional)
-    if structured_output["pages"]:
-        structured_output["pages"][0]["data"]["DocumentMetadata"] = textract_output.get("DocumentMetadata", {})
-
     return structured_output

tools/config.py

@@ -1,5 +1,12 @@
 import os
+import tempfile
+import socket
+from datetime import datetime
 from dotenv import load_dotenv
+from tldextract import TLDExtract
+
+today_rev = datetime.now().strftime("%Y%m%d")
+host_name = socket.gethostname()
 
 # Set or retrieve configuration variables for the redaction app
 
@@ -22,7 +29,7 @@ def get_or_create_env_var(var_name:str, default_value:str, print_val:bool=False)
 
 
 # If you have an aws_config env file in the config folder, you can load in app variables this way, e.g. '/env/app_config.env'
-APP_CONFIG_PATH = get_or_create_env_var('APP_CONFIG_PATH', '', print_val=True)
+APP_CONFIG_PATH = get_or_create_env_var('APP_CONFIG_PATH', '')
 
 
 if os.path.exists(APP_CONFIG_PATH):
@@ -34,7 +41,7 @@ if os.path.exists(APP_CONFIG_PATH):
 ###
 
 # If you have an aws_config env file in the config folder, you can load in AWS keys this way, e.g. '/env/aws_config.env'
-AWS_CONFIG_PATH = get_or_create_env_var('AWS_CONFIG_PATH', '', print_val=True)
+AWS_CONFIG_PATH = get_or_create_env_var('AWS_CONFIG_PATH', '')
 
 if os.path.exists(AWS_CONFIG_PATH):
     print(f"Loading AWS variables from config file {AWS_CONFIG_PATH}")
@@ -44,11 +51,11 @@ RUN_AWS_FUNCTIONS = get_or_create_env_var("RUN_AWS_FUNCTIONS", "0")
 
 AWS_REGION = get_or_create_env_var('AWS_REGION', 'eu-west-2')
 
-client_id = get_or_create_env_var('AWS_CLIENT_ID', '')
+AWS_CLIENT_ID = get_or_create_env_var('AWS_CLIENT_ID', '')
 
-client_secret = get_or_create_env_var('AWS_CLIENT_SECRET', '')
+AWS_CLIENT_SECRET = get_or_create_env_var('AWS_CLIENT_SECRET', '')
 
-user_pool_id = get_or_create_env_var('AWS_USER_POOL_ID', '')
+AWS_USER_POOL_ID = get_or_create_env_var('AWS_USER_POOL_ID', '')
 
 AWS_ACCESS_KEY = get_or_create_env_var('AWS_ACCESS_KEY', '')
 if AWS_ACCESS_KEY: print(f'AWS_ACCESS_KEY found in environment variables')
@@ -78,28 +85,53 @@ MAX_IMAGE_PIXELS = get_or_create_env_var('MAX_IMAGE_PIXELS', '') # Changed to No
 # File I/O config
 ###
 
-output_folder = get_or_create_env_var('GRADIO_OUTPUT_FOLDER', 'output/')
-print(f'The value of GRADIO_OUTPUT_FOLDER is {output_folder}')
+SESSION_OUTPUT_FOLDER = get_or_create_env_var('SESSION_OUTPUT_FOLDER', 'False') # i.e. do you want your input and output folders saved within a subfolder based on session hash value within output/input folders 
+
+OUTPUT_FOLDER = get_or_create_env_var('GRADIO_OUTPUT_FOLDER', 'output/') # 'output/'
+INPUT_FOLDER = get_or_create_env_var('GRADIO_INPUT_FOLDER', 'input/') # 'input/'
+
+# Allow for files to be saved in a temporary folder for increased security in some instances
+if OUTPUT_FOLDER == "TEMP" or INPUT_FOLDER == "TEMP": 
+    # Create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+        print(f'Temporary directory created at: {temp_dir}')
+
+        if OUTPUT_FOLDER == "TEMP": OUTPUT_FOLDER = temp_dir + "/"
+        if INPUT_FOLDER == "TEMP": INPUT_FOLDER = temp_dir + "/"
+
+FEEDBACK_LOGS_FOLDER = get_or_create_env_var('FEEDBACK_LOGS_FOLDER', 'feedback/' + today_rev + '/' + host_name + '/')
 
-session_output_folder = get_or_create_env_var('SESSION_OUTPUT_FOLDER', 'False')
-print(f'The value of SESSION_OUTPUT_FOLDER is {session_output_folder}')
+USAGE_LOGS_FOLDER = get_or_create_env_var('USAGE_LOGS_FOLDER', 'logs/' + today_rev + '/' + host_name + '/')
 
-input_folder = get_or_create_env_var('GRADIO_INPUT_FOLDER', 'input/')
-print(f'The value of GRADIO_INPUT_FOLDER is {input_folder}')
+ACCESS_LOGS_FOLDER = get_or_create_env_var('ACCESS_LOGS_FOLDER', 'usage/' + today_rev + '/' + host_name + '/')
+
+DISPLAY_FILE_NAMES_IN_LOGS = get_or_create_env_var('DISPLAY_FILE_NAMES_IN_LOGS', 'False')
 
 ###
 # REDACTION CONFIG
 ###
-# Number of pages to loop through before breaking the function and restarting from the last finished page.
-page_break_value = get_or_create_env_var('page_break_value', '50000')
+TESSERACT_FOLDER = get_or_create_env_var('TESSERACT_FOLDER', "tesseract/")
+
+POPPLER_FOLDER = get_or_create_env_var('POPPLER_FOLDER', "poppler/poppler-24.02.0/Library/bin/")
+
+SHOW_BULK_TEXTRACT_CALL_OPTIONS = get_or_create_env_var('SHOW_BULK_TEXTRACT_CALL_OPTIONS', 'False') # This feature not currently implemented
 
-max_time_value = get_or_create_env_var('max_time_value', '999999')
+# Number of pages to loop through before breaking the function and restarting from the last finished page (not currently activated).
+PAGE_BREAK_VALUE = get_or_create_env_var('PAGE_BREAK_VALUE', '99999')
+
+MAX_TIME_VALUE = get_or_create_env_var('MAX_TIME_VALUE', '999999')
 
 CUSTOM_BOX_COLOUR = get_or_create_env_var("CUSTOM_BOX_COLOUR", "")
 
+REDACTION_LANGUAGE = get_or_create_env_var("REDACTION_LANGUAGE", "en") # Currently only English is supported by the app
+
 ###
 # APP RUN CONFIG
 ###
+
+TLDEXTRACT_CACHE = get_or_create_env_var('TLDEXTRACT_CACHE', 'tld/.tld_set_snapshot')
+extract = TLDExtract(cache_dir=TLDEXTRACT_CACHE)
+
 # Get some environment variables and Launch the Gradio app
 COGNITO_AUTH = get_or_create_env_var('COGNITO_AUTH', '0')
 
@@ -117,4 +149,19 @@ DEFAULT_CONCURRENCY_LIMIT = get_or_create_env_var('DEFAULT_CONCURRENCY_LIMIT', '
 
 GET_DEFAULT_ALLOW_LIST = get_or_create_env_var('GET_DEFAULT_ALLOW_LIST', 'False')
 
-DEFAULT_ALLOW_LIST_PATH = get_or_create_env_var('DEFAULT_ALLOW_LIST_PATH', '')
+ALLOW_LIST_PATH = get_or_create_env_var('ALLOW_LIST_PATH', "config/default_allow_list.csv")
+
+S3_ALLOW_LIST_PATH = get_or_create_env_var('S3_ALLOW_LIST_PATH', '')
+
+SHOW_COSTS = get_or_create_env_var('SHOW_COSTS', 'True')
+
+GET_COST_CODES = get_or_create_env_var('GET_COST_CODES', 'False')
+
+COST_CODES_PATH = get_or_create_env_var('COST_CODES_PATH', 'config/COST_CENTRES.csv') # file should be a csv file with a single table in it that has two columns with a header. First column should contain cost codes, second column should contain a name or description for the cost code
+
+S3_COST_CODES_PATH = get_or_create_env_var('COST_CODES_PATH', '')
+
+ENFORCE_COST_CODES = get_or_create_env_var('ENFORCE_COST_CODES', 'False') # If you have cost codes listed, are they compulsory?
+
+if ENFORCE_COST_CODES == 'True': GET_COST_CODES = 'True'
+if GET_COST_CODES == 'True': ENFORCE_COST_CODES = 'False'

tools/data_anonymise.py

@@ -14,7 +14,7 @@ from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, DictAnalyzerR
 from presidio_anonymizer import AnonymizerEngine, BatchAnonymizerEngine
 from presidio_anonymizer.entities import OperatorConfig, ConflictResolutionStrategy
 
-from tools.config import RUN_AWS_FUNCTIONS, AWS_ACCESS_KEY, AWS_SECRET_KEY, output_folder
+from tools.config import RUN_AWS_FUNCTIONS, AWS_ACCESS_KEY, AWS_SECRET_KEY, OUTPUT_FOLDER
 from tools.helper_functions import get_file_name_without_type, read_file, detect_file_type
 from tools.load_spacy_model_custom_recognisers import nlp_analyser, score_threshold, custom_word_list_recogniser, CustomWordFuzzyRecognizer, custom_entities
 from tools.custom_image_analyser_engine import do_aws_comprehend_call
@@ -218,7 +218,7 @@ def anonymise_data_files(file_paths: List[str],
                          log_files_output_paths: list = [],
                          in_excel_sheets: list = [],
                          first_loop_state: bool = False,
-                         output_folder: str = output_folder,
+                         output_folder: str = OUTPUT_FOLDER,
                          in_deny_list:list[str]=[],
                          max_fuzzy_spelling_mistakes_num:int=0,
                          pii_identification_method:str="Local",
@@ -335,7 +335,7 @@ def anonymise_data_files(file_paths: List[str],
             file_type = ""
             out_file_part = anon_file
 
-            out_file_paths, out_message, key_string, log_files_output_paths = anon_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strat, language, chosen_redact_entities, in_allow_list, file_type, "", log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=output_folder)
+            out_file_paths, out_message, key_string, log_files_output_paths = anon_wrapper_func(anon_file, anon_df, chosen_cols, out_file_paths, out_file_part, out_message, sheet_name, anon_strat, language, chosen_redact_entities, in_allow_list, file_type, "", log_files_output_paths, in_deny_list, max_fuzzy_spelling_mistakes_num, pii_identification_method, chosen_redact_comprehend_entities, comprehend_query_number, comprehend_client, output_folder=OUTPUT_FOLDER)
         else:
             # If file is an xlsx, we are going to run through all the Excel sheets to anonymise them separately.
             file_type = detect_file_type(anon_file)
@@ -419,7 +419,7 @@ def anon_wrapper_func(
     chosen_redact_comprehend_entities:List[str]=[], 
     comprehend_query_number:int=0,
     comprehend_client:botocore.client.BaseClient="",
-    output_folder: str = output_folder
+    output_folder: str = OUTPUT_FOLDER
 ):
     """
     This function wraps the anonymisation process for a given dataframe. It filters the dataframe based on chosen columns, applies the specified anonymisation strategy using the anonymise_script function, and exports the anonymised data to a file.

tools/file_conversion.py

@@ -5,23 +5,28 @@ import os
 import re
 import time
 import json
+import numpy as np
 import pymupdf
-from pymupdf import Document
+from pymupdf import Document, Page, Rect
 import pandas as pd
-#import numpy as np
 import shutil
-from pymupdf import Rect
-from fitz import Page
+import zipfile
+from collections import defaultdict
 from tqdm import tqdm
 from gradio import Progress
-from typing import List, Optional
+from typing import List, Optional, Dict, Any
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pdf2image import convert_from_path
 from PIL import Image
 from scipy.spatial import cKDTree
 
-from tools.config import output_folder, IMAGES_DPI, LOAD_TRUNCATED_IMAGES, MAX_IMAGE_PIXELS, CUSTOM_BOX_COLOUR
+IMAGE_NUM_REGEX = re.compile(r'_(\d+)\.png$')
+
+pd.set_option('future.no_silent_downcasting', True)
+
+from tools.config import OUTPUT_FOLDER, INPUT_FOLDER, IMAGES_DPI, LOAD_TRUNCATED_IMAGES, MAX_IMAGE_PIXELS, CUSTOM_BOX_COLOUR
 from tools.helper_functions import get_file_name_without_type, tesseract_ocr_option, text_ocr_option, textract_option, read_file
+# from tools.aws_textract import load_and_convert_textract_json
 
 image_dpi = float(IMAGES_DPI)
 if not MAX_IMAGE_PIXELS: Image.MAX_IMAGE_PIXELS = None
@@ -56,16 +61,16 @@ def is_pdf(filename):
     """
     return filename.lower().endswith(".pdf")
 
-# %%
 ## Convert pdf to image if necessary
 
-
-
 def check_image_size_and_reduce(out_path:str, image:Image):
     '''
     Check if a given image size is above around 4.5mb, and reduce size if necessary. 5mb is the maximum possible to submit to AWS Textract.
     '''
 
+    all_img_details = []
+    page_num = 0
+
     # Check file size and resize if necessary
     max_size = 4.5 * 1024 * 1024  # 5 MB in bytes # 5
     file_size = os.path.getsize(out_path)        
@@ -93,61 +98,82 @@ def check_image_size_and_reduce(out_path:str, image:Image):
             print(f"Resized to {new_width}x{new_height}, new file_size: {file_size}")
     else:
         new_width = width
-        new_height = height 
+        new_height = height
+    
+    
+    all_img_details.append((page_num, image, new_width, new_height))
 
-    return new_width, new_height
+    return image, new_width, new_height, all_img_details, out_path
 
-def process_single_page(pdf_path: str, page_num: int, image_dpi: float, output_dir: str = 'input') -> tuple[int, str]:
-    try:
-        # Construct the full output directory path
-        output_dir = os.path.join(os.getcwd(), output_dir)
-        out_path = os.path.join(output_dir, f"{os.path.basename(pdf_path)}_{page_num}.png")
-        os.makedirs(os.path.dirname(out_path), exist_ok=True)
-
-        if os.path.exists(out_path):
-            # Load existing image
-            image = Image.open(out_path)
-        else:
-            # Convert PDF page to image
-            image_l = convert_from_path(pdf_path, first_page=page_num+1, last_page=page_num+1, 
-                                        dpi=image_dpi, use_cropbox=False, use_pdftocairo=False)
-            image = image_l[0]
-            image = image.convert("L")
-            image.save(out_path, format="PNG")
+def process_single_page_for_image_conversion(pdf_path:str, page_num:int, image_dpi:float=image_dpi, create_images:bool = True, input_folder: str = INPUT_FOLDER) -> tuple[int, str, float, float]:
+
+    out_path_placeholder = "placeholder_image_" + str(page_num) + ".png"
 
-        width, height = image.size
+    if create_images == True:
+        try:
+            # Construct the full output directory path
+            image_output_dir = os.path.join(os.getcwd(), input_folder)
+            out_path = os.path.join(image_output_dir, f"{os.path.basename(pdf_path)}_{page_num}.png")
+            os.makedirs(os.path.dirname(out_path), exist_ok=True)
 
-        # Check if image size too large and reduce if necessary
-        width, height = check_image_size_and_reduce(out_path, image)                
+            if os.path.exists(out_path):
+                # Load existing image
+                image = Image.open(out_path)
+            elif pdf_path.lower().endswith(".pdf"):
+                # Convert PDF page to image
+                image_l = convert_from_path(pdf_path, first_page=page_num+1, last_page=page_num+1, 
+                                            dpi=image_dpi, use_cropbox=False, use_pdftocairo=False)
+                image = image_l[0]
+                image = image.convert("L")
 
-        return page_num, out_path, width, height
+                image.save(out_path, format="PNG")
+            elif pdf_path.lower().endswith(".jpg") or pdf_path.lower().endswith(".png") or pdf_path.lower().endswith(".jpeg"):
+                image = Image.open(pdf_path)
+                image.save(out_path, format="PNG")
 
-    except Exception as e:
-        print(f"Error processing page {page_num + 1}: {e}")
-        return page_num, "", width, height
+            width, height = image.size
 
-def convert_pdf_to_images(pdf_path: str, prepare_for_review:bool=False, page_min: int = 0, image_dpi: float = image_dpi, num_threads: int = 8, output_dir: str = '/input'):
+            # Check if image size too large and reduce if necessary
+            #print("Checking size of image and reducing if necessary.")
+            image, width, height, all_img_details, img_path = check_image_size_and_reduce(out_path, image)                
 
-    # If preparing for review, just load the first page (not used)
+            return page_num, out_path, width, height
+
+        except Exception as e:
+            print(f"Error processing page {page_num + 1}: {e}")
+            return page_num,  out_path_placeholder, pd.NA, pd.NA
+    else:
+        # print("Not creating image for page", page_num)
+        return page_num,  out_path_placeholder, pd.NA, pd.NA
+
+def convert_pdf_to_images(pdf_path: str, prepare_for_review:bool=False, page_min: int = 0, page_max:int = 0, create_images:bool=True, image_dpi: float = image_dpi, num_threads: int = 8, input_folder: str = INPUT_FOLDER):
+
+    # If preparing for review, just load the first page (not currently used)
     if prepare_for_review == True:
         page_count = pdfinfo_from_path(pdf_path)['Pages'] #1
+        page_min = 0
+        page_max = page_count
     else:
         page_count = pdfinfo_from_path(pdf_path)['Pages']
 
     print(f"Number of pages in PDF: {page_count}")
 
+    # Set page max to length of pdf if not specified
+    if page_max == 0: page_max = page_count
+
     results = []
     with ThreadPoolExecutor(max_workers=num_threads) as executor:
         futures = []
-        for page_num in range(page_min, page_count):
-            futures.append(executor.submit(process_single_page, pdf_path, page_num, image_dpi))
+        for page_num in range(page_min, page_max):
+            futures.append(executor.submit(process_single_page_for_image_conversion, pdf_path, page_num, image_dpi, create_images=create_images, input_folder=input_folder))
         
-        for future in tqdm(as_completed(futures), total=len(futures), unit="pages", desc="Converting pages"):
-            page_num, result, width, height = future.result()
-            if result:
-                results.append((page_num, result, width, height))
+        for future in tqdm(as_completed(futures), total=len(futures), unit="pages", desc="Converting pages to image"):
+            page_num, img_path, width, height = future.result()
+            if img_path:
+                results.append((page_num, img_path, width, height))
             else:
                 print(f"Page {page_num + 1} failed to process.")
+                results.append((page_num, "placeholder_image_" + str(page_num) + ".png", pd.NA, pd.NA))
     
     # Sort results by page number
     results.sort(key=lambda x: x[0])
@@ -156,10 +182,10 @@ def convert_pdf_to_images(pdf_path: str, prepare_for_review:bool=False, page_min
     heights = [result[3] for result in results]
 
     print("PDF has been converted to images.")
-    return images, widths, heights
+    return images, widths, heights, results
 
 # Function to take in a file path, decide if it is an image or pdf, then process appropriately.
-def process_file(file_path:str, prepare_for_review:bool=False):
+def process_file_for_image_creation(file_path:str, prepare_for_review:bool=False, input_folder:str=INPUT_FOLDER, create_images:bool=True):
     # Get the file extension
     file_extension = os.path.splitext(file_path)[1].lower()
  
@@ -171,21 +197,30 @@ def process_file(file_path:str, prepare_for_review:bool=False):
 
         # Load images from the file paths. Test to see if it is bigger than 4.5 mb and reduct if needed (Textract limit is 5mb)
         image = Image.open(file_path)
-        img_object, image_sizes_width, image_sizes_height = check_image_size_and_reduce(file_path, image)
+        img_object, image_sizes_width, image_sizes_height, all_img_details, img_path = check_image_size_and_reduce(file_path, image)
+
+        if not isinstance(image_sizes_width, list):
+            img_path = [img_path]
+            image_sizes_width = [image_sizes_width]
+            image_sizes_height = [image_sizes_height]
+            all_img_details = [all_img_details]
+            
 
     # Check if the file is a PDF
     elif file_extension == '.pdf':
         print(f"{file_path} is a PDF file. Converting to image set")
+
         # Run your function for processing PDF files here
-        img_object, image_sizes_width, image_sizes_height = convert_pdf_to_images(file_path, prepare_for_review)
+        img_path, image_sizes_width, image_sizes_height, all_img_details = convert_pdf_to_images(file_path, prepare_for_review, input_folder=input_folder, create_images=create_images)
 
     else:
         print(f"{file_path} is not an image or PDF file.")
-        img_object = []
+        img_path = []
         image_sizes_width = []
         image_sizes_height = []
+        all_img_details = []
 
-    return img_object, image_sizes_width, image_sizes_height
+    return img_path, image_sizes_width, image_sizes_height, all_img_details
 
 def get_input_file_names(file_input:List[str]):
     '''
@@ -195,8 +230,8 @@ def get_input_file_names(file_input:List[str]):
     all_relevant_files = []
     file_name_with_extension = ""
     full_file_name = ""
+    total_pdf_page_count = 0
 
-    #print("file_input in input file names:", file_input)
     if isinstance(file_input, dict):
         file_input = os.path.abspath(file_input["name"])
 
@@ -215,23 +250,38 @@ def get_input_file_names(file_input:List[str]):
 
         file_extension = os.path.splitext(file_path)[1].lower()
 
-        # Check if the file is an image type
+        # Check if the file is in acceptable types
         if (file_extension in ['.jpg', '.jpeg', '.png', '.pdf', '.xlsx', '.csv', '.parquet']) & ("review_file" not in file_path_without_ext):
             all_relevant_files.append(file_path_without_ext)
             file_name_with_extension = file_path_without_ext + file_extension
             full_file_name = file_path
+
+        # If PDF, get number of pages
+        if (file_extension in ['.pdf']):
+            # Open the PDF file
+            pdf_document = pymupdf.open(file_path)
+            # Get the number of pages
+            page_count = pdf_document.page_count
+            
+            # Close the document
+            pdf_document.close()
+        else:
+            page_count = 1
+
+        total_pdf_page_count += page_count
     
     all_relevant_files_str = ", ".join(all_relevant_files)
 
-    #print("all_relevant_files_str in input_file_names", all_relevant_files_str)
-    #print("all_relevant_files in input_file_names", all_relevant_files)
-
-    return all_relevant_files_str, file_name_with_extension, full_file_name, all_relevant_files
+    return all_relevant_files_str, file_name_with_extension, full_file_name, all_relevant_files, total_pdf_page_count
 
 def convert_color_to_range_0_1(color):
     return tuple(component / 255 for component in color)
 
 def redact_single_box(pymupdf_page:Page, pymupdf_rect:Rect, img_annotation_box:dict, custom_colours:bool=False):
+    '''
+    Commit redaction boxes to a PyMuPDF page.
+    '''
+
     pymupdf_x1 = pymupdf_rect[0]
     pymupdf_y1 = pymupdf_rect[1]
     pymupdf_x2 = pymupdf_rect[2]
@@ -247,7 +297,6 @@ def redact_single_box(pymupdf_page:Page, pymupdf_rect:Rect, img_annotation_box:d
         redact_bottom_y = middle_y - 1
         redact_top_y = middle_y + 1
 
-    #print("Rect:", rect)
 
     rect_small_pixel_height = Rect(pymupdf_x1, redact_bottom_y, pymupdf_x2, redact_top_y)  # Slightly smaller than outside box
 
@@ -274,8 +323,7 @@ def redact_single_box(pymupdf_page:Page, pymupdf_rect:Rect, img_annotation_box:d
     #shape.finish(color=(0, 0, 0))  # Black fill for the rectangle
     shape.commit()
 
-
-def convert_pymupdf_to_image_coords(pymupdf_page, x1, y1, x2, y2, image: Image):
+def convert_pymupdf_to_image_coords(pymupdf_page:Page, x1:float, y1:float, x2:float, y2:float, image: Image=None, image_dimensions:dict={}):
     '''
     Converts coordinates from pymupdf format to image coordinates,
     accounting for mediabox dimensions and offset.
@@ -291,22 +339,17 @@ def convert_pymupdf_to_image_coords(pymupdf_page, x1, y1, x2, y2, image: Image):
     mediabox_height = mediabox.height
     
     # Get target image dimensions
-    image_page_width, image_page_height = image.size
+    if image:
+        image_page_width, image_page_height = image.size
+    elif image_dimensions:
+        image_page_width, image_page_height = image_dimensions['image_width'], image_dimensions['image_height']
+    else:
+        image_page_width, image_page_height = mediabox_width, mediabox_height
 
     # Calculate scaling factors
     image_to_mediabox_x_scale = image_page_width / mediabox_width
     image_to_mediabox_y_scale = image_page_height / mediabox_height
 
-    image_to_rect_scale_width = image_page_width / rect_width
-    image_to_rect_scale_height = image_page_height / rect_height
-
-    # Adjust for offsets (difference in position between mediabox and rect)
-    x_offset = rect.x0 - mediabox.x0  # Difference in x position
-    y_offset = rect.y0 - mediabox.y0  # Difference in y position
-
-    #print("x_offset:", x_offset)
-    #print("y_offset:", y_offset)
-
     # Adjust coordinates:
     # Apply scaling to match image dimensions
     x1_image = x1 * image_to_mediabox_x_scale    
@@ -339,24 +382,24 @@ def convert_pymupdf_to_image_coords(pymupdf_page, x1, y1, x2, y2, image: Image):
 
     return x1_image, y1_image, x2_image, y2_image
 
-def redact_whole_pymupdf_page(rect_height, rect_width, image, page, custom_colours, border = 5):
+def redact_whole_pymupdf_page(rect_height:float, rect_width:float, image:Image, page:Page, custom_colours, border:float = 5, image_dimensions:dict={}):
     # Small border to page that remains white
     border = 5
     # Define the coordinates for the Rect
     whole_page_x1, whole_page_y1 = 0 + border, 0 + border  # Bottom-left corner
     whole_page_x2, whole_page_y2 = rect_width - border, rect_height - border  # Top-right corner
 
-    whole_page_image_x1, whole_page_image_y1, whole_page_image_x2, whole_page_image_y2 = convert_pymupdf_to_image_coords(page, whole_page_x1, whole_page_y1, whole_page_x2, whole_page_y2, image)
+    # whole_page_image_x1, whole_page_image_y1, whole_page_image_x2, whole_page_image_y2 = convert_pymupdf_to_image_coords(page, whole_page_x1, whole_page_y1, whole_page_x2, whole_page_y2, image, image_dimensions=image_dimensions)
 
     # Create new image annotation element based on whole page coordinates
     whole_page_rect = Rect(whole_page_x1, whole_page_y1, whole_page_x2, whole_page_y2)
 
     # Write whole page annotation to annotation boxes
     whole_page_img_annotation_box = {}
-    whole_page_img_annotation_box["xmin"] = whole_page_image_x1
-    whole_page_img_annotation_box["ymin"] = whole_page_image_y1
-    whole_page_img_annotation_box["xmax"] = whole_page_image_x2
-    whole_page_img_annotation_box["ymax"] = whole_page_image_y2
+    whole_page_img_annotation_box["xmin"] = whole_page_x1 #whole_page_image_x1
+    whole_page_img_annotation_box["ymin"] = whole_page_y1 #whole_page_image_y1
+    whole_page_img_annotation_box["xmax"] = whole_page_x2 #whole_page_image_x2
+    whole_page_img_annotation_box["ymax"] =  whole_page_y2 #whole_page_image_y2
     whole_page_img_annotation_box["color"] = (0,0,0)
     whole_page_img_annotation_box["label"] = "Whole page"
 
@@ -364,7 +407,7 @@ def redact_whole_pymupdf_page(rect_height, rect_width, image, page, custom_colou
 
     return whole_page_img_annotation_box
 
-def create_page_size_objects(pymupdf_doc:Document, image_sizes_width:List[float], image_sizes_height:List[float]):
+def create_page_size_objects(pymupdf_doc:Document, image_sizes_width:List[float], image_sizes_height:List[float], image_file_paths:List[str]):
     page_sizes = []
     original_cropboxes = []
 
@@ -377,9 +420,9 @@ def create_page_size_objects(pymupdf_doc:Document, image_sizes_width:List[float]
         # Create a page_sizes_object.
         # If images have been created, then image width an height come from this value. Otherwise, they are set to the cropbox size
         if image_sizes_width and image_sizes_height:
-            out_page_image_sizes = {"page":reported_page_no, "image_width":image_sizes_width[page_no], "image_height":image_sizes_height[page_no], "mediabox_width":pymupdf_page.mediabox.width, "mediabox_height": pymupdf_page.mediabox.height, "cropbox_width":pymupdf_page.cropbox.width, "cropbox_height":pymupdf_page.cropbox.height}
+            out_page_image_sizes = {"page":reported_page_no, "image_path":image_file_paths[page_no], "image_width":image_sizes_width[page_no], "image_height":image_sizes_height[page_no], "mediabox_width":pymupdf_page.mediabox.width, "mediabox_height": pymupdf_page.mediabox.height, "cropbox_width":pymupdf_page.cropbox.width, "cropbox_height":pymupdf_page.cropbox.height, "original_cropbox":original_cropboxes[-1]}
         else:
-            out_page_image_sizes = {"page":reported_page_no, "image_width":pd.NA(), "image_height":pd.NA(), "mediabox_width":pymupdf_page.mediabox.width, "mediabox_height": pymupdf_page.mediabox.height, "cropbox_width":pymupdf_page.cropbox.width, "cropbox_height":pymupdf_page.cropbox.height}
+            out_page_image_sizes = {"page":reported_page_no, "image_path":image_file_paths[page_no], "image_width":pd.NA, "image_height":pd.NA, "mediabox_width":pymupdf_page.mediabox.width, "mediabox_height": pymupdf_page.mediabox.height, "cropbox_width":pymupdf_page.cropbox.width, "cropbox_height":pymupdf_page.cropbox.height, "original_cropbox":original_cropboxes[-1]}
         
         page_sizes.append(out_page_image_sizes)
 
@@ -395,8 +438,11 @@ def prepare_image_or_pdf(
     all_annotations_object:List = [],
     prepare_for_review:bool = False,
     in_fully_redacted_list:List[int]=[],
-    output_folder:str=output_folder,
+    output_folder:str=OUTPUT_FOLDER,
+    input_folder:str=INPUT_FOLDER,
     prepare_images:bool=True,
+    page_sizes:list[dict]=[],
+    textract_output_found:bool = False,    
     progress: Progress = Progress(track_tqdm=True)
 ) -> tuple[List[str], List[str]]:
     """
@@ -416,7 +462,9 @@ def prepare_image_or_pdf(
         prepare_for_review(optional, bool): Is this preparation step preparing pdfs and json files to review current redactions?
         in_fully_redacted_list(optional, List of int): A list of pages to fully redact
         output_folder (optional, str): The output folder for file save
-        prepare_images (optional, bool): A boolean indicating whether to create images for each PDF page. Defaults to true
+        prepare_images (optional, bool): A boolean indicating whether to create images for each PDF page. Defaults to True.
+        page_sizes(optional, List[dict]): A list of dicts containing information about page sizes in various formats.
+        textract_output_found (optional, bool): A boolean indicating whether textract output has already been found . Defaults to False.
         progress (optional, Progress): Progress tracker for the operation
         
 
@@ -430,6 +478,7 @@ def prepare_image_or_pdf(
     converted_file_paths = []
     image_file_paths = []
     pymupdf_doc = []
+    all_img_details = []    
     review_file_csv = pd.DataFrame()
 
     if isinstance(in_fully_redacted_list, pd.DataFrame):
@@ -438,39 +487,21 @@ def prepare_image_or_pdf(
 
     # If this is the first time around, set variables to 0/blank
     if first_loop_state==True:
-        print("first_loop_state is True")
         latest_file_completed = 0
         out_message = []
         all_annotations_object = []
     else:
-        print("Now attempting file:", str(latest_file_completed))
-    
-    # This is only run when a new page is loaded, so can reset page loop values. If end of last file (99), current loop number set to 999
-    # if latest_file_completed == 99:
-    #     current_loop_page_number = 999
-    #     page_break_return = False
-    # else:
-    #     current_loop_page_number = 0
-    #     page_break_return = False
-
+        print("Now redacting file", str(latest_file_completed))
+  
     # If out message or converted_file_paths are blank, change to a list so it can be appended to
-    if isinstance(out_message, str):
-        out_message = [out_message]  
+    if isinstance(out_message, str): out_message = [out_message]
 
-    if not file_paths:
-        file_paths = []
+    if not file_paths: file_paths = []
 
-    if isinstance(file_paths, dict):
-        file_paths = os.path.abspath(file_paths["name"])
+    if isinstance(file_paths, dict): file_paths = os.path.abspath(file_paths["name"])
 
-    if isinstance(file_paths, str):
-        file_path_number = 1
-    else:
-        file_path_number = len(file_paths)
-
-    #print("Current_loop_page_number at start of prepare_image_or_pdf function is:", current_loop_page_number)
-    print("Number of file paths:", file_path_number)
-    print("Latest_file_completed:", latest_file_completed)
+    if isinstance(file_paths, str): file_path_number = 1
+    else: file_path_number = len(file_paths)
     
     latest_file_completed = int(latest_file_completed)
 
@@ -481,9 +512,7 @@ def prepare_image_or_pdf(
             final_out_message = '\n'.join(out_message)
         else:
             final_out_message = out_message
-        return final_out_message, converted_file_paths, image_file_paths, number_of_pages, number_of_pages, pymupdf_doc, all_annotations_object, review_file_csv, original_cropboxes, page_sizes
-
-    #in_allow_list_flat = [item for sublist in in_allow_list for item in sublist]
+        return final_out_message, converted_file_paths, image_file_paths, number_of_pages, number_of_pages, pymupdf_doc, all_annotations_object, review_file_csv, original_cropboxes, page_sizes, textract_output_found, all_img_details
 
     progress(0.1, desc='Preparing file')
 
@@ -524,33 +553,14 @@ def prepare_image_or_pdf(
             pymupdf_doc = pymupdf.open(file_path)
             pymupdf_pages = pymupdf_doc.page_count
 
-            # Load cropbox dimensions to use later  
-
             converted_file_path = file_path
 
             if prepare_images==True:
-                image_file_paths, image_sizes_width, image_sizes_height = process_file(file_path, prepare_for_review)
+                image_file_paths, image_sizes_width, image_sizes_height, all_img_details = process_file_for_image_creation(file_path, prepare_for_review, input_folder, create_images=True)
             else:
-                print("Skipping image preparation")
-                image_file_paths=[]
-                image_sizes_width=[]
-                image_sizes_height=[]
-
-            # Create page sizes object
-            # page_sizes = []
-
-            # for i, page in enumerate(pymupdf_doc):
-            #     page_no = i
-            #     reported_page_no = i + 1
-                
-            #     pymupdf_page = pymupdf_doc.load_page(page_no)
-            #     original_cropboxes.append(pymupdf_page.cropbox)  # Save original CropBox
-
-            #     # Create a page_sizes_object
-            #     out_page_image_sizes = {"page":reported_page_no, "image_width":image_sizes_width[page_no], "image_height":image_sizes_height[page_no], "mediabox_width":pymupdf_page.mediabox.width, "mediabox_height": pymupdf_page.mediabox.height, "cropbox_width":pymupdf_page.cropbox.width, "cropbox_height":pymupdf_page.cropbox.height}
-            #     page_sizes.append(out_page_image_sizes)
-
-            page_sizes, original_cropboxes = create_page_size_objects(pymupdf_doc, image_sizes_width, image_sizes_height)
+                image_file_paths, image_sizes_width, image_sizes_height, all_img_details = process_file_for_image_creation(file_path, prepare_for_review, input_folder, create_images=False)
+            
+            page_sizes, original_cropboxes = create_page_size_objects(pymupdf_doc, image_sizes_width, image_sizes_height, image_file_paths)
 
             #Create base version of the annotation object that doesn't have any annotations in it
             if (not all_annotations_object) & (prepare_for_review == True):
@@ -577,22 +587,17 @@ def prepare_image_or_pdf(
             pymupdf_page.insert_image(rect, filename=file_path)  # Insert the image into the page
             pymupdf_page = pymupdf_doc.load_page(0)
 
-            original_cropboxes.append(pymupdf_page.cropbox)  # Save original CropBox
-
             file_path_str = str(file_path)
 
-            image_file_paths, image_sizes_width, image_sizes_height = process_file(file_path_str, prepare_for_review)
+            image_file_paths, image_sizes_width, image_sizes_height, all_img_details = process_file_for_image_creation(file_path_str, prepare_for_review, input_folder, create_images=True)
+
 
-            #print("image_file_paths:", image_file_paths)
             # Create a page_sizes_object
-            out_page_image_sizes = {"page":1, "image_width":image_sizes_width[0], "image_height":image_sizes_height[0], "mediabox_width":pymupdf_page.mediabox.width, "mediabox_height": pymupdf_page.mediabox.height, "cropbox_width":original_cropboxes[-1].width, "cropbox_height":original_cropboxes[-1].height}
-            page_sizes.append(out_page_image_sizes)
+            page_sizes, original_cropboxes = create_page_size_objects(pymupdf_doc, image_sizes_width, image_sizes_height, image_file_paths)
 
             converted_file_path = output_folder + file_name_with_ext
 
-            pymupdf_doc.save(converted_file_path)
-
-            print("Inserted image into PDF file")
+            pymupdf_doc.save(converted_file_path, garbage=4, deflate=True, clean=True)
 
         elif file_extension in ['.csv']:
             review_file_csv = read_file(file)
@@ -604,7 +609,6 @@ def prepare_image_or_pdf(
         if (file_extension in ['.json']) | (json_from_csv == True):
 
             if (file_extension in ['.json']) &  (prepare_for_review == True):
-                print("Preparing file for review")
                 if isinstance(file_path, str):
                     with open(file_path, 'r') as json_file:
                         all_annotations_object = json.load(json_file)
@@ -614,18 +618,18 @@ def prepare_image_or_pdf(
 
             # Assume it's a textract json
             elif (file_extension == '.json') and (prepare_for_review is not True):
-                # If the file ends with textract.json, assume it's a Textract response object.
                 # Copy it to the output folder so it can be used later.
-                out_folder = os.path.join(output_folder, file_path_without_ext + ".json")
+                out_textract_path = os.path.join(output_folder, file_path_without_ext + "_textract.json")
 
                 # Use shutil to copy the file directly
-                shutil.copy2(file_path, out_folder)  # Preserves metadata
+                shutil.copy2(file_path, out_textract_path)  # Preserves metadata
+
+                textract_output_found = True
                 
                 continue
 
             # If you have an annotations object from the above code
             if all_annotations_object:
-                #print("out_annotations_object before reloading images:", all_annotations_object)
 
                 # Get list of page numbers
                 image_file_paths_pages = [
@@ -637,11 +641,6 @@ def prepare_image_or_pdf(
                 
                 # If PDF pages have been converted to image files, replace the current image paths in the json to this. 
                 if image_file_paths:
-                    #print("Image file paths found")
-
-                    #print("Image_file_paths:", image_file_paths)
-
-                    #for i, annotation in enumerate(all_annotations_object):
                     for i, image_file_path in enumerate(image_file_paths):
 
                         if i < len(all_annotations_object): 
@@ -650,18 +649,15 @@ def prepare_image_or_pdf(
                             annotation = {}
                             all_annotations_object.append(annotation)
 
-                        #print("annotation:", annotation, "for page:", str(i))
                         try:
                             if not annotation:
                                 annotation = {"image":"", "boxes": []}
                                 annotation_page_number = int(re.search(r'_(\d+)\.png$', image_file_path).group(1))
-
                             else:
                                 annotation_page_number = int(re.search(r'_(\d+)\.png$', annotation["image"]).group(1))
                         except Exception as e:
                             print("Extracting page number from image failed due to:", e)
                             annotation_page_number = 0
-                        #print("Annotation page number:", annotation_page_number)
 
                         # Check if the annotation page number exists in the image file paths pages
                         if annotation_page_number in image_file_paths_pages:
@@ -674,26 +670,53 @@ def prepare_image_or_pdf(
 
                         all_annotations_object[i] = annotation
 
-                    #print("all_annotations_object at end of json/csv load part:", all_annotations_object)
+                
+                if isinstance(in_fully_redacted_list, list):
+                    in_fully_redacted_list = pd.DataFrame(data={"fully_redacted_pages_list":in_fully_redacted_list})
 
                 # Get list of pages that are to be fully redacted and redact them
-                # if not in_fully_redacted_list.empty:
-                #     print("Redacting whole pages")
+                if not in_fully_redacted_list.empty:
+                    print("Redacting whole pages")
 
-                #     for i, image in enumerate(image_file_paths):
-                #         page = pymupdf_doc.load_page(i)
-                #         rect_height = page.rect.height
-                #         rect_width = page.rect.width 
-                #         whole_page_img_annotation_box = redact_whole_pymupdf_page(rect_height, rect_width, image, page, custom_colours = False, border = 5)
+                    for i, image in enumerate(image_file_paths):
+                        page = pymupdf_doc.load_page(i)
+                        rect_height = page.rect.height
+                        rect_width = page.rect.width 
+                        whole_page_img_annotation_box = redact_whole_pymupdf_page(rect_height, rect_width, image, page, custom_colours = False, border = 5, image_dimensions={"image_width":image_sizes_width[i], "image_height":image_sizes_height[i]})
 
-                #         all_annotations_object.append(whole_page_img_annotation_box)
+                        all_annotations_object.append(whole_page_img_annotation_box)
 
                 # Write the response to a JSON file in output folder
                 out_folder = output_folder + file_path_without_ext + ".json"
-                with open(out_folder, 'w') as json_file:
-                    json.dump(all_annotations_object, json_file, indent=4)  # indent=4 makes the JSON file pretty-printed
+                # with open(out_folder, 'w') as json_file:
+                #     json.dump(all_annotations_object, json_file, separators=(",", ":"))
                 continue
 
+        # If it's a zip, it could be extract from a Textract bulk API call. Check it's this, and load in json if found
+        elif file_extension in ['.zip']:
+
+            # Assume it's a Textract response object. Copy it to the output folder so it can be used later.
+            out_folder = os.path.join(output_folder, file_path_without_ext + "_textract.json")
+
+            # Use shutil to copy the file directly
+            # Open the ZIP file to check its contents
+            with zipfile.ZipFile(file_path, 'r') as zip_ref:
+                json_files = [f for f in zip_ref.namelist() if f.lower().endswith('.json')]
+
+                if len(json_files) == 1:  # Ensure only one JSON file exists
+                    json_filename = json_files[0]
+
+                    # Extract the JSON file to the same directory as the ZIP file
+                    extracted_path = os.path.join(os.path.dirname(file_path), json_filename)
+                    zip_ref.extract(json_filename, os.path.dirname(file_path))
+
+                    # Move the extracted JSON to the intended output location
+                    shutil.move(extracted_path, out_folder)
+
+                    textract_output_found = True
+                else:
+                    print(f"Skipping {file_path}: Expected 1 JSON file, found {len(json_files)}")
+
         # Must be something else, return with error message
         else:
             if in_redact_method == tesseract_ocr_option or in_redact_method == textract_option:
@@ -706,8 +729,7 @@ def prepare_image_or_pdf(
                 if is_pdf(file_path) == False:
                     out_message = "Please upload a PDF file for text analysis."
                     print(out_message)
-                    raise Exception(out_message)  
-
+                    raise Exception(out_message)
 
         converted_file_paths.append(converted_file_path)
         image_file_paths.extend(image_file_path)        
@@ -722,29 +744,23 @@ def prepare_image_or_pdf(
 
     number_of_pages = len(image_file_paths)
         
-    return out_message_out, converted_file_paths, image_file_paths, number_of_pages, number_of_pages, pymupdf_doc, all_annotations_object, review_file_csv, original_cropboxes, page_sizes
+    return out_message_out, converted_file_paths, image_file_paths, number_of_pages, number_of_pages, pymupdf_doc, all_annotations_object, review_file_csv, original_cropboxes, page_sizes, textract_output_found, all_img_details
 
-def convert_text_pdf_to_img_pdf(in_file_path:str, out_text_file_path:List[str], image_dpi:float=image_dpi):
+def convert_text_pdf_to_img_pdf(in_file_path:str, out_text_file_path:List[str], image_dpi:float=image_dpi, output_folder:str=OUTPUT_FOLDER, input_folder:str=INPUT_FOLDER):
     file_path_without_ext = get_file_name_without_type(in_file_path)
 
     out_file_paths = out_text_file_path
 
-    # Convert annotated text pdf back to image to give genuine redactions
-    print("Creating image version of redacted PDF to embed redactions.")
-    
-    pdf_text_image_paths, image_sizes_width, image_sizes_height = process_file(out_text_file_path[0])
+    # Convert annotated text pdf back to image to give genuine redactions   
+    pdf_text_image_paths, image_sizes_width, image_sizes_height, all_img_details = process_file_for_image_creation(out_file_paths[0], input_folder=input_folder)
     out_text_image_file_path = output_folder + file_path_without_ext + "_text_redacted_as_img.pdf"
     pdf_text_image_paths[0].save(out_text_image_file_path, "PDF" ,resolution=image_dpi, save_all=True, append_images=pdf_text_image_paths[1:])
 
-    # out_file_paths.append(out_text_image_file_path)
-
     out_file_paths = [out_text_image_file_path]
 
     out_message = "PDF " + file_path_without_ext + " converted to image-based file."
     print(out_message)
 
-    #print("Out file paths:", out_file_paths)
-
     return out_message, out_file_paths
 
 def join_values_within_threshold(df1:pd.DataFrame, df2:pd.DataFrame):
@@ -775,212 +791,595 @@ def join_values_within_threshold(df1:pd.DataFrame, df2:pd.DataFrame):
 
     # Clean up extra columns
     final_df = final_df.drop(columns=['key'])
-    print(final_df)
-
 
-def convert_annotation_json_to_review_df(all_annotations:List[dict], redaction_decision_output:pd.DataFrame=pd.DataFrame(), page_sizes:List[dict]=[]) -> pd.DataFrame:
+def remove_duplicate_images_with_blank_boxes(data: List[dict]) -> List[dict]:
     '''
-    Convert the annotation json data to a dataframe format. Add on any text from the initial review_file dataframe by joining on pages/co-ordinates (doesn't work very well currently).
+    Remove items from the annotator object where the same page exists twice.
     '''
-    # Flatten the data
-    flattened_annotation_data = []
-    page_sizes_df = pd.DataFrame()
-
-    if not isinstance(redaction_decision_output, pd.DataFrame):
-        redaction_decision_output = pd.DataFrame()
-
-    for annotation in all_annotations:
-        #print("annotation:", annotation)
-        #print("flattened_data:", flattened_data)
-        image_path = annotation["image"]
-
-        # Use regex to find the number before .png
-        match = re.search(r'_(\d+)\.png$', image_path)
-        if match:
-            number = match.group(1)  # Extract the number
-            #print(number)  # Output: 0
-            reported_number = int(number) + 1
+    # Group items by 'image'
+    image_groups = defaultdict(list)
+    for item in data:
+        image_groups[item['image']].append(item)
+
+    # Process each group to prioritize items with non-empty boxes
+    result = []
+    for image, items in image_groups.items():
+        # Filter items with non-empty boxes
+        non_empty_boxes = [item for item in items if item.get('boxes')]
+
+         # Remove 'text' elements from boxes
+        for item in non_empty_boxes:
+            if 'boxes' in item:
+                item['boxes'] = [{k: v for k, v in box.items() if k != 'text'} for box in item['boxes']]
+
+        if non_empty_boxes:
+            # Keep the first entry with non-empty boxes
+            result.append(non_empty_boxes[0])
         else:
-            print("No number found before .png. Returning page 1.")
-            reported_number = 1
+            # If all items have empty or missing boxes, keep the first item
+            result.append(items[0])
 
-        # Check if 'boxes' is in the annotation, if not, add an empty list
-        if 'boxes' not in annotation:
-            annotation['boxes'] = []        
+    return result
 
-        for box in annotation["boxes"]:
-            if 'text' not in box:
-                data_to_add = {"image": image_path, "page": reported_number,  **box} # "text": annotation['text'],
-            else:
-                data_to_add = {"image": image_path, "page": reported_number, "text": box['text'], **box}
-            #print("data_to_add:", data_to_add)
-            flattened_annotation_data.append(data_to_add)
+def divide_coordinates_by_page_sizes(review_file_df:pd.DataFrame, page_sizes_df:pd.DataFrame, xmin="xmin", xmax="xmax", ymin="ymin", ymax="ymax"):
 
-    # Convert to a DataFrame
-    review_file_df = pd.DataFrame(flattened_annotation_data)
+    '''Convert data to same coordinate system. If all coordinates all greater than one, this is a absolute image coordinates - change back to relative coordinates.'''
 
-    if page_sizes:
-        page_sizes_df = pd.DataFrame(page_sizes)
-        page_sizes_df["page"] = page_sizes_df["page"].astype(int)
+    review_file_df_out = review_file_df
 
-    # Convert data to same coordinate system
-    # If all coordinates all greater than one, this is a absolute image coordinates - change back to relative coordinates
-    if "xmin" in review_file_df.columns:
-        if review_file_df["xmin"].max() >= 1 and review_file_df["xmax"].max() >= 1 and review_file_df["ymin"].max() >= 1 and review_file_df["ymax"].max() >= 1:
-            #print("review file df has large coordinates")
-            review_file_df["page"] = review_file_df["page"].astype(int)
+    if xmin in review_file_df.columns and not review_file_df.empty:
+        review_file_df_orig = review_file_df.copy().loc[(review_file_df[xmin] <= 1) & (review_file_df[xmax] <= 1) & (review_file_df[ymin] <= 1) & (review_file_df[ymax] <= 1),:]
+        
+        review_file_df = review_file_df.loc[(review_file_df[xmin] > 1) & (review_file_df[xmax] > 1) & (review_file_df[ymin] > 1) & (review_file_df[ymax] > 1),:]
 
-            if "image_width" not in review_file_df.columns and not page_sizes_df.empty:                            
-                review_file_df = review_file_df.merge(page_sizes_df, on="page", how="left")
+        review_file_df.loc[:, "page"] = pd.to_numeric(review_file_df["page"], errors="coerce")
 
-            if "image_width" in review_file_df.columns:
-                #print("Dividing coordinates in review file")
-                review_file_df["xmin"] = review_file_df["xmin"] / review_file_df["image_width"]
-                review_file_df["xmax"] = review_file_df["xmax"] / review_file_df["image_width"]
-                review_file_df["ymin"] = review_file_df["ymin"] / review_file_df["image_height"]
-                review_file_df["ymax"] = review_file_df["ymax"] / review_file_df["image_height"]
+        review_file_df_div = review_file_df
 
-                #print("review_file_df after coordinates divided:", review_file_df)
+        if "image_width" not in review_file_df_div.columns and not page_sizes_df.empty:  
 
-    if not redaction_decision_output.empty:
-        # If all coordinates all greater than one, this is a absolute image coordinates - change back to relative coordinates
-        if redaction_decision_output["xmin"].max() >= 1 and redaction_decision_output["xmax"].max() >= 1 and redaction_decision_output["ymin"].max() >= 1 and redaction_decision_output["ymax"].max() >= 1:
+            page_sizes_df["image_width"] = page_sizes_df["image_width"].replace("<NA>", pd.NA)
+            page_sizes_df["image_height"] = page_sizes_df["image_height"].replace("<NA>", pd.NA)
+            review_file_df_div = review_file_df_div.merge(page_sizes_df[["page", "image_width", "image_height", "mediabox_width", "mediabox_height"]], on="page", how="left")
 
-            redaction_decision_output["page"] = redaction_decision_output["page"].astype(int)
+        if "image_width" in review_file_df_div.columns:
+            if review_file_df_div["image_width"].isna().all():  # Check if all are NaN values. If so, assume we only have mediabox coordinates available
+                review_file_df_div["image_width"] = review_file_df_div["image_width"].fillna(review_file_df_div["mediabox_width"]).infer_objects()
+                review_file_df_div["image_height"] = review_file_df_div["image_height"].fillna(review_file_df_div["mediabox_height"]).infer_objects()
 
-            if "image_width" not in redaction_decision_output.columns and not page_sizes_df.empty:                            
-                redaction_decision_output = redaction_decision_output.merge(page_sizes_df, on="page", how="left")
+            convert_type_cols = ["image_width", "image_height", xmin, xmax, ymin, ymax]
+            review_file_df_div[convert_type_cols] = review_file_df_div[convert_type_cols].apply(pd.to_numeric, errors="coerce")  
 
-            if "image_width" in redaction_decision_output.columns:
-                redaction_decision_output["xmin"] = redaction_decision_output["xmin"] / redaction_decision_output["image_width"]
-                redaction_decision_output["xmax"] = redaction_decision_output["xmax"] / redaction_decision_output["image_width"]
-                redaction_decision_output["ymin"] = redaction_decision_output["ymin"] / redaction_decision_output["image_height"]
-                redaction_decision_output["ymax"] = redaction_decision_output["ymax"] / redaction_decision_output["image_height"]
+            review_file_df_div[xmin] = review_file_df_div[xmin] / review_file_df_div["image_width"]
+            review_file_df_div[xmax] = review_file_df_div[xmax] / review_file_df_div["image_width"]
+            review_file_df_div[ymin] = review_file_df_div[ymin] / review_file_df_div["image_height"]
+            review_file_df_div[ymax] = review_file_df_div[ymax] / review_file_df_div["image_height"]
 
-    #print("convert_review_json review_file_df before merges:", review_file_df[['xmin', 'ymin', 'xmax', 'ymax', 'label']])
-    #print("review_file_df[xmin]", review_file_df["xmin"])
+        # Concatenate the original and modified DataFrames
+        dfs_to_concat = [df for df in [review_file_df_orig, review_file_df_div] if not df.empty]
+        if dfs_to_concat:  # Ensure there's at least one non-empty DataFrame
+            review_file_df_out = pd.concat(dfs_to_concat)
+        else:
+            review_file_df_out = review_file_df  # Return an original DataFrame instead of raising an error
 
-    #print("redaction_decision_output:", redaction_decision_output)
-    #print("review_file_df:", review_file_df)
+        # Only sort if the DataFrame is not empty and contains the required columns
+        required_sort_columns = {"page", xmin, ymin}
+        if not review_file_df_out.empty and required_sort_columns.issubset(review_file_df_out.columns):
+            review_file_df_out.sort_values(["page", ymin, xmin], inplace=True)
 
-    # Join on additional text data from decision output results if included, if text not already there
-    if not redaction_decision_output.empty: 
-        if not 'text' in redaction_decision_output.columns:
-            redaction_decision_output['text'] = ''
+    review_file_df_out.drop(["image_width", "image_height", "mediabox_width", "mediabox_height"], axis=1, errors="ignore")
 
-        if not 'text' in review_file_df.columns:
-            review_file_df['text'] = ''
+    return review_file_df_out
 
-        # Load DataFrames
-        df1 = review_file_df.copy()
-        df2 = redaction_decision_output.copy()
+def multiply_coordinates_by_page_sizes(review_file_df: pd.DataFrame, page_sizes_df: pd.DataFrame, xmin="xmin", xmax="xmax", ymin="ymin", ymax="ymax"):
 
-        #print("review_file before tolerance merge:", review_file_df)
-        #print("redaction_decision_output before tolerance merge:", redaction_decision_output)
 
-        # Create a unique key based on coordinates and label for exact merge
-        merge_keys = ['xmin', 'ymin', 'xmax', 'ymax', 'label', 'page']
-        df1['key'] = df1[merge_keys].astype(str).agg('_'.join, axis=1)
-        df2['key'] = df2[merge_keys].astype(str).agg('_'.join, axis=1)
+    if xmin in review_file_df.columns and not review_file_df.empty:
+        # Separate absolute vs relative coordinates
+        review_file_df_orig = review_file_df.loc[
+            (review_file_df[xmin] > 1) & (review_file_df[xmax] > 1) & 
+            (review_file_df[ymin] > 1) & (review_file_df[ymax] > 1), :].copy()
 
-        # Attempt exact merge first
-        #merged_df = df1.merge(df2[['key', 'text']], on='key', how='left')
+        review_file_df = review_file_df.loc[
+            (review_file_df[xmin] <= 1) & (review_file_df[xmax] <= 1) & 
+            (review_file_df[ymin] <= 1) & (review_file_df[ymax] <= 1), :].copy()
 
-        # Attempt exact merge first, renaming df2['text'] to avoid suffixes
-        merged_df = df1.merge(df2[['key', 'text']], on='key', how='left', suffixes=('', '_duplicate'))
+        if review_file_df.empty:
+            return review_file_df_orig  # If nothing is left, return the original absolute-coordinates DataFrame
 
-        # If a match is found, keep that text; otherwise, keep the original df1 text
-        merged_df['text'] = merged_df['text'].combine_first(merged_df.pop('text_duplicate'))
+        review_file_df.loc[:, "page"] = pd.to_numeric(review_file_df["page"], errors="coerce")
 
-        #print("merged_df['text']:", merged_df['text'])
+        if "image_width" not in review_file_df.columns and not page_sizes_df.empty:
+            page_sizes_df[['image_width', 'image_height']] = page_sizes_df[['image_width','image_height']].replace("<NA>", pd.NA)  # Ensure proper NA handling
+            review_file_df = review_file_df.merge(page_sizes_df, on="page", how="left")
+
+        if "image_width" in review_file_df.columns:
+            # Split into rows with/without image size info
+            review_file_df_not_na = review_file_df.loc[review_file_df["image_width"].notna()].copy()
+            review_file_df_na = review_file_df.loc[review_file_df["image_width"].isna()].copy()
+
+            if not review_file_df_not_na.empty:
+                convert_type_cols = ["image_width", "image_height", xmin, xmax, ymin, ymax]
+                review_file_df_not_na[convert_type_cols] = review_file_df_not_na[convert_type_cols].apply(pd.to_numeric, errors="coerce")
+
+                # Multiply coordinates by image sizes
+                review_file_df_not_na[xmin] *= review_file_df_not_na["image_width"]
+                review_file_df_not_na[xmax] *= review_file_df_not_na["image_width"]
+                review_file_df_not_na[ymin] *= review_file_df_not_na["image_height"]
+                review_file_df_not_na[ymax] *= review_file_df_not_na["image_height"]
+
+            # Concatenate the modified and unmodified data
+            review_file_df = pd.concat([df for df in [review_file_df_not_na, review_file_df_na] if not df.empty])
+
+        # Merge with the original absolute-coordinates DataFrame
+        dfs_to_concat = [df for df in [review_file_df_orig, review_file_df] if not df.empty]
+        if dfs_to_concat:  # Ensure there's at least one non-empty DataFrame
+            review_file_df = pd.concat(dfs_to_concat)
+        else:
+            review_file_df = pd.DataFrame()  # Return an empty DataFrame instead of raising an error
+
+        # Only sort if the DataFrame is not empty and contains the required columns
+        required_sort_columns = {"page", "xmin", "ymin"}
+        if not review_file_df.empty and required_sort_columns.issubset(review_file_df.columns):
+            review_file_df.sort_values(["page", "xmin", "ymin"], inplace=True)
+
+    return review_file_df
+
+
+def do_proximity_match_by_page_for_text(df1:pd.DataFrame, df2:pd.DataFrame):
+    '''
+    Match text from one dataframe to another based on proximity matching of coordinates page by page.
+    '''
+
+    if not 'text' in df2.columns: df2['text'] = ''
+    if not 'text' in df1.columns: df1['text'] = ''
+
+    # Create a unique key based on coordinates and label for exact merge
+    merge_keys = ['xmin', 'ymin', 'xmax', 'ymax', 'label', 'page']
+    df1['key'] = df1[merge_keys].astype(str).agg('_'.join, axis=1)
+    df2['key'] = df2[merge_keys].astype(str).agg('_'.join, axis=1)
+
+    # Attempt exact merge first
+    merged_df = df1.merge(df2[['key', 'text']], on='key', how='left', suffixes=('', '_duplicate'))
+
+    # If a match is found, keep that text; otherwise, keep the original df1 text
+    merged_df['text'] = np.where(
+        merged_df['text'].isna() | (merged_df['text'] == ''),
+        merged_df.pop('text_duplicate'),
+        merged_df['text']
+    )
+
+    # Define tolerance for proximity matching
+    tolerance = 0.02
+
+    # Precompute KDTree for each page in df2
+    page_trees = {}
+    for page in df2['page'].unique():
+        df2_page = df2[df2['page'] == page]
+        coords = df2_page[['xmin', 'ymin', 'xmax', 'ymax']].values
+        if np.all(np.isfinite(coords)) and len(coords) > 0:
+            page_trees[page] = (cKDTree(coords), df2_page)
+
+    # Perform proximity matching
+    for i, row in df1.iterrows():
+        page_number = row['page']
+
+        if page_number in page_trees:
+            tree, df2_page = page_trees[page_number]
+
+            # Query KDTree for nearest neighbor
+            dist, idx = tree.query([row[['xmin', 'ymin', 'xmax', 'ymax']].values], distance_upper_bound=tolerance)
+
+            if dist[0] < tolerance and idx[0] < len(df2_page):
+                merged_df.at[i, 'text'] = df2_page.iloc[idx[0]]['text']
+
+    # Drop the temporary key column
+    merged_df.drop(columns=['key'], inplace=True)
+
+    return merged_df
+
+
+def do_proximity_match_all_pages_for_text(df1:pd.DataFrame, df2:pd.DataFrame, threshold:float=0.03):
+    '''
+    Match text from one dataframe to another based on proximity matching of coordinates across all pages.
+    '''
+
+    if not 'text' in df2.columns: df2['text'] = ''
+    if not 'text' in df1.columns: df1['text'] = ''
+
+    # Create a unique key based on coordinates and label for exact merge
+    merge_keys = ['xmin', 'ymin', 'xmax', 'ymax', 'label', 'page']
+    df1['key'] = df1[merge_keys].astype(str).agg('_'.join, axis=1)
+    df2['key'] = df2[merge_keys].astype(str).agg('_'.join, axis=1)
+
+    # Attempt exact merge first, renaming df2['text'] to avoid suffixes
+    merged_df = df1.merge(df2[['key', 'text']], on='key', how='left', suffixes=('', '_duplicate'))
+
+    # If a match is found, keep that text; otherwise, keep the original df1 text
+    merged_df['text'] = np.where(
+        merged_df['text'].isna() | (merged_df['text'] == ''),
+        merged_df.pop('text_duplicate'),
+        merged_df['text']
+    )
+
+    # Handle missing matches using a proximity-based approach
+    # Convert coordinates to numpy arrays for KDTree lookup
+    query_coords = np.array(df1[['xmin', 'ymin', 'xmax', 'ymax']].values, dtype=float)
+
+    # Check for NaN or infinite values in query_coords and filter them out
+    finite_mask = np.isfinite(query_coords).all(axis=1)
+    if not finite_mask.all():
+        print("Warning: query_coords contains non-finite values. Filtering out non-finite entries.")
+        query_coords = query_coords[finite_mask]  # Filter out rows with NaN or infinite values
+    else:
+        pass
+    
+    # Proceed only if query_coords is not empty
+    if query_coords.size > 0:
+        # Ensure df2 is filtered for finite values before creating the KDTree
+        finite_mask_df2 = np.isfinite(df2[['xmin', 'ymin', 'xmax', 'ymax']].values).all(axis=1)
+        df2_finite = df2[finite_mask_df2]
+
+        # Create the KDTree with the filtered data
+        tree = cKDTree(df2_finite[['xmin', 'ymin', 'xmax', 'ymax']].values)
 
-        # Handle missing matches using a proximity-based approach
-        #if merged_df['text'].isnull().sum() > 0:
-        #print("Attempting tolerance-based merge for text")
-        # Convert coordinates to numpy arrays for KDTree lookup
-        tree = cKDTree(df2[['xmin', 'ymin', 'xmax', 'ymax']].values)
-        query_coords = df1[['xmin', 'ymin', 'xmax', 'ymax']].values
-        
         # Find nearest neighbors within a reasonable tolerance (e.g., 1% of page)
-        tolerance = 0.01
+        tolerance = threshold
         distances, indices = tree.query(query_coords, distance_upper_bound=tolerance)
 
         # Assign text values where matches are found
         for i, (dist, idx) in enumerate(zip(distances, indices)):
-            if dist < tolerance and idx < len(df2):
-                merged_df.at[i, 'text'] = df2.iloc[idx]['text']
+            if dist < tolerance and idx < len(df2_finite):
+                merged_df.at[i, 'text'] = df2_finite.iloc[idx]['text']
+
+    # Drop the temporary key column
+    merged_df.drop(columns=['key'], inplace=True)
+
+    return merged_df
+
+
 
-        # Drop the temporary key column
-        merged_df.drop(columns=['key'], inplace=True)
 
-        review_file_df = merged_df
+def _extract_page_number(image_path: Any) -> int:
+    """Helper function to safely extract page number."""
+    if not isinstance(image_path, str):
+        return 1
+    match = IMAGE_NUM_REGEX.search(image_path)
+    if match:
+        try:
+            return int(match.group(1)) + 1
+        except (ValueError, TypeError):
+            return 1
+    return 1
 
-        review_file_df = review_file_df[["image", "page", "label", "color", "xmin", "ymin", "xmax", "ymax", "text"]]
+def convert_annotation_data_to_dataframe(all_annotations: List[Dict[str, Any]]):
+    '''
+    Convert annotation list to DataFrame using Pandas explode and json_normalize.
+    '''
+    if not all_annotations:
+        # Return an empty DataFrame with the expected schema if input is empty
+        return pd.DataFrame(columns=["image", "page", "xmin", "xmax", "ymin", "ymax", "text"])
+
+    # 1. Create initial DataFrame from the list of annotations
+    # Use list comprehensions with .get() for robustness
+    df = pd.DataFrame({
+        "image": [anno.get("image") for anno in all_annotations],
+        # Ensure 'boxes' defaults to an empty list if missing or None
+        "boxes": [anno.get("boxes") if isinstance(anno.get("boxes"), list) else [] for anno in all_annotations]
+    })
+
+    # 2. Calculate the page number using the helper function
+    df['page'] = df['image'].apply(_extract_page_number)
+
+    # 3. Handle empty 'boxes' lists *before* exploding.
+    # Explode removes rows where the list is empty. We want to keep them
+    # as rows with NA values. Replace empty lists with a list containing
+    # a single placeholder dictionary.
+    placeholder_box = {"xmin": pd.NA, "xmax": pd.NA, "ymin": pd.NA, "ymax": pd.NA, "text": pd.NA}
+    df['boxes'] = df['boxes'].apply(lambda x: x if x else [placeholder_box])
+
+    # 4. Explode the 'boxes' column. Each item in the list becomes a new row.
+    df_exploded = df.explode('boxes', ignore_index=True)
+
+    # 5. Normalize the 'boxes' column (which now contains dictionaries or the placeholder)
+    # This turns the dictionaries into separate columns.
+    # Check for NaNs or non-dict items just in case, though placeholder handles most cases.
+    mask = df_exploded['boxes'].notna() & df_exploded['boxes'].apply(isinstance, args=(dict,))
+    normalized_boxes = pd.json_normalize(df_exploded.loc[mask, 'boxes'])
+
+    # 6. Combine the base data (image, page) with the normalized box data
+    # Use the index of the exploded frame (where mask is True) to ensure correct alignment
+    final_df = df_exploded.loc[mask, ['image', 'page']].reset_index(drop=True).join(normalized_boxes)
+
+    # --- Optional: Handle rows that might have had non-dict items in 'boxes' ---
+    # If there were rows filtered out by 'mask', you might want to add them back
+    # with NA values for box columns. However, the placeholder strategy usually
+    # prevents this from being necessary.
+
+    # 7. Ensure essential columns exist and set column order
+    essential_box_cols = ["xmin", "xmax", "ymin", "ymax", "text"]
+    for col in essential_box_cols:
+        if col not in final_df.columns:
+            final_df[col] = pd.NA # Add column with NA if it wasn't present in any box
+
+    base_cols = ["image", "page"]
+    extra_box_cols = [col for col in final_df.columns if col not in base_cols and col not in essential_box_cols]
+    final_col_order = base_cols + essential_box_cols + sorted(extra_box_cols)
+
+    # Reindex to ensure consistent column order and presence of essential columns
+    # Using fill_value=pd.NA isn't strictly needed here as we added missing columns above,
+    # but it's good practice if columns could be missing for other reasons.
+    final_df = final_df.reindex(columns=final_col_order, fill_value=pd.NA)
+
+    return final_df
+
+
+# def convert_annotation_data_to_dataframe(all_annotations:List[dict]):
+#         '''
+#         Convert an annotation list of dictionaries to a dataframe with all boxes on a separate row
+#         '''
+#         # Flatten the data
+#         flattened_annotation_data = []        
+
+#         for annotation in all_annotations:
+#             image_path = annotation["image"]
+
+#             if image_path:
+#                 match = re.search(r'_(\d+)\.png$', image_path)
+#                 if match:
+#                     number = match.group(1)
+#                     reported_number = int(number) + 1
+#                 else:
+#                     reported_number = 1
+#             else:
+#                 reported_number = 1
+
+#             # Check if 'boxes' is in the annotation, if not, add an empty list
+#             if 'boxes' not in annotation:
+#                 annotation['boxes'] = []
+
+#             # If boxes are empty, create a row with blank values for xmin, xmax, ymin, ymax
+#             if not annotation["boxes"]:
+#                 data_to_add = {"image": image_path, "page": reported_number, "xmin": pd.NA, "xmax": pd.NA, "ymin": pd.NA, "ymax": pd.NA}
+#                 flattened_annotation_data.append(data_to_add)
+#             else:
+#                 for box in annotation["boxes"]:
+#                     if 'xmin' not in box:
+#                         data_to_add = {"image": image_path, "page": reported_number, "xmin": pd.NA, 'xmax': pd.NA, 'ymin': pd.NA, 'ymax': pd.NA}
+#                     elif 'text' not in box:
+#                         data_to_add = {"image": image_path, "page": reported_number, **box}
+#                     else:
+#                         data_to_add = {"image": image_path, "page": reported_number, "text": box['text'], **box}
+#                     flattened_annotation_data.append(data_to_add)
+
+#         # Convert to a DataFrame
+#         review_file_df = pd.DataFrame(flattened_annotation_data)
+
+#         return review_file_df
+
+# def create_annotation_dicts_from_annotation_df(all_image_annotations_df:pd.DataFrame, page_sizes:List[dict]):
+#         '''
+#         From an annotation object as a dataframe, convert back to a list of dictionaries that can be used in the Gradio Image Annotator component
+#         '''
+#         result = []
+
+#         # Ensure that every page has an entry in the resulting list of dicts
+#         for image_path in page_sizes:
+#             annotation = {}
+#             annotation["image"] = image_path["image_path"]
+#             annotation["boxes"] = []
+
+#             result.append(annotation)
+
+#         # Then add in all the filled in data
+#         for image, group in all_image_annotations_df.groupby('image'):
+#             boxes = group[['xmin', 'ymin', 'xmax', 'ymax', 'color', 'label']].to_dict(orient='records')
+#             result.append({'image': image, 'boxes': boxes})
+        
+#         return result
+
+def create_annotation_dicts_from_annotation_df(
+    all_image_annotations_df: pd.DataFrame,
+    page_sizes: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    '''
+    Convert annotation DataFrame back to list of dicts using dictionary lookup.
+    Ensures all images from page_sizes are present without duplicates.
+    '''
+    # 1. Create a dictionary keyed by image path for efficient lookup & update
+    # Initialize with all images from page_sizes. Use .get for safety.
+    image_dict: Dict[str, Dict[str, Any]] = {}
+    for item in page_sizes:
+        image_path = item.get("image_path")
+        if image_path:  # Only process if image_path exists and is not None/empty
+            image_dict[image_path] = {"image": image_path, "boxes": []}
+
+    # Check if the DataFrame is empty or lacks necessary columns
+    if all_image_annotations_df.empty or 'image' not in all_image_annotations_df.columns:
+        print("Warning: Annotation DataFrame is empty or missing 'image' column.")
+        return list(image_dict.values()) # Return based on page_sizes only
+
+    # 2. Define columns to extract for boxes and check availability
+    # Make sure these columns actually exist in the DataFrame
+    box_cols = ['xmin', 'ymin', 'xmax', 'ymax', 'color', 'label']
+    available_cols = [col for col in box_cols if col in all_image_annotations_df.columns]
+
+    if not available_cols:
+        print(f"Warning: None of the expected box columns ({box_cols}) found in DataFrame.")
+        return list(image_dict.values()) # Return based on page_sizes only
+
+    # 3. Group the DataFrame by image and update the dictionary
+    # Drop rows where essential coordinates might be NA (adjust if NA is meaningful)
+    coord_cols = ['xmin', 'ymin', 'xmax', 'ymax']
+    valid_box_df = all_image_annotations_df.dropna(
+        subset=[col for col in coord_cols if col in available_cols]
+    ).copy() # Use .copy() to avoid SettingWithCopyWarning if modifying later
+
+
+    # Check if any valid boxes remain after dropping NAs
+    if valid_box_df.empty:
+         print("Warning: No valid annotation rows found in DataFrame after dropping NA coordinates.")
+         return list(image_dict.values())
+
+
+    # Process groups
+    try:
+        for image_path, group in valid_box_df.groupby('image', observed=True, sort=False):
+            # Check if this image path exists in our target dictionary (from page_sizes)
+            if image_path in image_dict:
+                # Convert the relevant columns of the group to a list of dicts
+                # Using only columns that are actually available
+                boxes = group[available_cols].to_dict(orient='records')
+                # Update the 'boxes' list in the dictionary
+                image_dict[image_path]['boxes'] = boxes
+            # Else: Image found in DataFrame but not required by page_sizes; ignore it.
+    except KeyError:
+        # This shouldn't happen due to the 'image' column check above, but handle defensively
+        print("Error: Issue grouping DataFrame by 'image'.")
+        return list(image_dict.values())
+
+
+    # 4. Convert the dictionary values back into the final list format
+    result = list(image_dict.values())
+
+    return result
+
+# import pandas as pd
+# from typing import List, Dict, Any
+
+# def create_annotation_dicts_from_annotation_df(
+#     all_image_annotations_df: pd.DataFrame,
+#     page_sizes: List[Dict[str, Any]]
+# ) -> List[Dict[str, Any]]:
+#     '''
+#     Convert annotation DataFrame back to list of dicts using Pandas merge.
+#     Ensures all images from page_sizes are present without duplicates.
+#     '''
+#     # 1. Create a DataFrame containing all required image paths from page_sizes
+#     if not page_sizes:
+#         return []
+#     all_image_paths = [item.get("image_path") for item in page_sizes if item.get("image_path")]
+#     if not all_image_paths:
+#         return []
+#     # Use unique paths
+#     pages_df = pd.DataFrame({'image': list(set(all_image_paths))})
+
+#     # Check if the DataFrame is empty or lacks necessary columns
+#     if all_image_annotations_df.empty or 'image' not in all_image_annotations_df.columns:
+#         print("Warning: Annotation DataFrame is empty or missing 'image' column.")
+#         # Add empty boxes column and return
+#         pages_df['boxes'] = [[] for _ in range(len(pages_df))]
+#         return pages_df.to_dict(orient='records')
+
+#     # 2. Define columns to extract and check availability
+#     box_cols = ['xmin', 'ymin', 'xmax', 'ymax', 'color', 'label']
+#     available_cols = [col for col in box_cols if col in all_image_annotations_df.columns]
+
+#     if not available_cols:
+#         print(f"Warning: None of the expected box columns ({box_cols}) found in DataFrame.")
+#         pages_df['boxes'] = [[] for _ in range(len(pages_df))]
+#         return pages_df.to_dict(orient='records')
+
+#     # 3. Prepare the annotation data: drop invalid rows and aggregate boxes
+#     coord_cols = ['xmin', 'ymin', 'xmax', 'ymax']
+#     valid_box_df = all_image_annotations_df.dropna(
+#         subset=[col for col in coord_cols if col in available_cols]
+#     ).copy() # Use .copy()
+
+#     if valid_box_df.empty:
+#          print("Warning: No valid annotation rows found after dropping NA coordinates.")
+#          pages_df['boxes'] = [[] for _ in range(len(pages_df))]
+#          return pages_df.to_dict(orient='records')
+
+
+#     # Aggregate boxes into lists of dictionaries per image
+#     def aggregate_boxes(group):
+#         return group[available_cols].to_dict(orient='records')
+
+#     # Group by image and apply the aggregation
+#     grouped_boxes = valid_box_df.groupby('image', observed=True, sort=False).apply(aggregate_boxes).reset_index(name='boxes')
+
+#     # 4. Perform a left merge: keep all images from pages_df, add boxes where they exist
+#     merged_df = pd.merge(pages_df, grouped_boxes, on='image', how='left')
+
+#     # 5. Fill NaN in 'boxes' column (for images with no annotations) with empty lists
+#     # Ensure the column exists before trying to fillna
+#     if 'boxes' in merged_df.columns:
+#          # Use apply with a lambda for robust filling of NAs or potential None values
+#          merged_df['boxes'] = merged_df['boxes'].apply(lambda x: [] if pd.isna(x) else x)
+#     else:
+#          # Should not happen with left merge, but handle defensively
+#          merged_df['boxes'] = [[] for _ in range(len(merged_df))]
+
+
+#     # 6. Convert the final DataFrame to the list of dictionaries format
+#     result = merged_df.to_dict(orient='records')
+
+#     return result
+
+def convert_annotation_json_to_review_df(all_annotations:List[dict],
+                                         redaction_decision_output:pd.DataFrame=pd.DataFrame(),
+                                         page_sizes:pd.DataFrame=pd.DataFrame(),
+                                         do_proximity_match:bool=True) -> pd.DataFrame:
+    '''
+    Convert the annotation json data to a dataframe format. Add on any text from the initial review_file dataframe by joining on pages/co-ordinates (if option selected).
+    '''
+    
+    review_file_df = convert_annotation_data_to_dataframe(all_annotations)
+    
+    if page_sizes:
+        page_sizes_df = pd.DataFrame(page_sizes)
+        page_sizes_df[["page"]] = page_sizes_df[["page"]].apply(pd.to_numeric, errors="coerce")        
+    
+        review_file_df = divide_coordinates_by_page_sizes(review_file_df, page_sizes_df)
+
+        redaction_decision_output = divide_coordinates_by_page_sizes(redaction_decision_output, page_sizes_df)
+
+    # Join on additional text data from decision output results if included, if text not already there
+    if not redaction_decision_output.empty and not review_file_df.empty and do_proximity_match == True:       
+
+        # Match text to review file to match on text
+        review_file_df = do_proximity_match_all_pages_for_text(df1 = review_file_df.copy(), df2 = redaction_decision_output.copy())
 
     # Ensure required columns exist, filling with blank if they don't
-    for col in ["image", "page", "label", "color", "xmin", "ymin", "xmax", "ymax", "text"]:
+    check_columns = ["image", "page", "label", "color", "xmin", "ymin", "xmax", "ymax", "text"]
+
+    for col in check_columns:
         if col not in review_file_df.columns:
             review_file_df[col] = ''
 
-    #for col in ['xmin', 'xmax', 'ymin', 'ymax']:
-    #    review_file_df[col] = np.floor(review_file_df[col])
+    if not review_file_df.empty:
+        review_file_df = review_file_df[check_columns]
+    else:
+        review_file_df = pd.DataFrame(columns=check_columns)
 
     # If colours are saved as list, convert to tuple
     review_file_df.loc[:,"color"] = review_file_df.loc[:,"color"].apply(lambda x: tuple(x) if isinstance(x, list) else x)
 
-    # print("page_sizes:", page_sizes)
-
-    # Convert page sizes to relative values
-    # if page_sizes:
-    #     print("Checking page sizes")
-        
-    #     page_sizes_df = pd.DataFrame(page_sizes)
-
-    #     if "image_width" not in review_file_df.columns:
-    #         review_file_df = review_file_df.merge(page_sizes_df, how="left", on = "page")
-        
-    #     # If all coordinates all greater than one, this is a absolute image coordinates - change back to relative coordinates
-    #     if review_file_df["xmin"].max() > 1 and review_file_df["xmax"].max() > 1 and review_file_df["ymin"].max() > 1 and review_file_df["ymax"].max() > 1:
-    #         print("Dividing coordinates by image width and height.")
-    #         review_file_df["xmin"] = review_file_df["xmin"] / review_file_df["image_width"]
-    #         review_file_df["xmax"] = review_file_df["xmax"] / review_file_df["image_width"]
-    #         review_file_df["ymin"] = review_file_df["ymin"] / review_file_df["image_height"]
-    #         review_file_df["ymax"] = review_file_df["ymax"] / review_file_df["image_height"]
-
     review_file_df = review_file_df.sort_values(['page', 'ymin', 'xmin', 'label'])
 
-    #review_file_df.to_csv(output_folder + "review_file_test.csv", index=None)
-
     return review_file_df
 
-def convert_review_df_to_annotation_json(review_file_df:pd.DataFrame, image_paths:List[Image.Image], page_sizes:List[dict]=[]) -> List[dict]:
+def convert_review_df_to_annotation_json(review_file_df:pd.DataFrame,
+                                         image_paths:List[Image.Image],
+                                         page_sizes:List[dict]=[]) -> List[dict]:
     '''
     Convert a review csv to a json file for use by the Gradio Annotation object.
     '''
+    # Make sure all relevant cols are float
+    float_cols = ["page", "xmin", "xmax", "ymin", "ymax"]
+    for col in float_cols:
+        review_file_df.loc[:, col] = pd.to_numeric(review_file_df.loc[:, col], errors='coerce')
     
     # Convert relative co-ordinates into image coordinates for the image annotation output object
     if page_sizes:        
         page_sizes_df = pd.DataFrame(page_sizes)
+        page_sizes_df[["page"]] = page_sizes_df[["page"]].apply(pd.to_numeric, errors="coerce")
 
-        # If there are no image coordinates, then just convert the first page to image to be able to see this at least.
-        if len(page_sizes_df.loc[page_sizes_df["image_width"].isnull(),"image_width"]) == len(page_sizes_df["image_width"]):
-            print("No image dimensions found, converting first page.")
-
-        # If no nulls, then can do image coordinate conversion
-        elif len(page_sizes_df.loc[page_sizes_df["image_width"].isnull(),"image_width"]) == 0:
-
-            if "image_width" not in review_file_df.columns:            
-                    review_file_df = review_file_df.merge(page_sizes_df, how="left", on = "page")
-            
-            # If all coordinates are less or equal to one, this is a relative page scaling - change back to image coordinates
-            if review_file_df["xmin"].max() <= 1 and review_file_df["xmax"].max() <= 1 and review_file_df["ymin"].max() <= 1 and review_file_df["ymax"].max() <= 1:
-                review_file_df["xmin"] = review_file_df["xmin"] * review_file_df["image_width"]
-                review_file_df["xmax"] = review_file_df["xmax"] * review_file_df["image_width"]
-                review_file_df["ymin"] = review_file_df["ymin"] * review_file_df["image_height"]
-                review_file_df["ymax"] = review_file_df["ymax"] * review_file_df["image_height"]
-            
+        review_file_df = multiply_coordinates_by_page_sizes(review_file_df, page_sizes_df)
+           
     # Keep only necessary columns
-    review_file_df = review_file_df[["image", "page", "xmin", "ymin", "xmax", "ymax", "color", "label"]]
+    review_file_df = review_file_df[["image", "page", "label", "color", "xmin", "ymin", "xmax", "ymax"]].drop_duplicates(subset=["image", "page", "xmin", "ymin", "xmax", "ymax", "label"])
 
     # If colours are saved as list, convert to tuple
     review_file_df.loc[:, "color"] = review_file_df.loc[:,"color"].apply(lambda x: tuple(x) if isinstance(x, list) else x)
@@ -991,16 +1390,15 @@ def convert_review_df_to_annotation_json(review_file_df:pd.DataFrame, image_path
     # Create a list to hold the JSON data
     json_data = []
 
-    for page_no, pdf_image_path in enumerate(image_paths):
-        reported_page_number = int(page_no + 1)            
+    for page_no, pdf_image_path in enumerate(page_sizes_df["image_path"]):
+        
+        reported_page_number = int(page_no + 1)
 
         if reported_page_number in review_file_df["page"].values:
 
             # Convert each relevant group to a list of box dictionaries
             selected_csv_pages = grouped_csv_pages.get_group(reported_page_number)
             annotation_boxes = selected_csv_pages.drop(columns=['image', 'page']).to_dict(orient='records')
-
-            # If all bbox coordinates are below 1, then they are relative. Need to convert based on image size.
             
             annotation = {
                 "image": pdf_image_path,

tools/find_duplicate_pages.py

@@ -3,36 +3,32 @@ import pandas as pd
 #import glob
 import os
 import re
-from tools.helper_functions import output_folder
+from tools.helper_functions import OUTPUT_FOLDER
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
-import nltk
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
-from nltk.stem import PorterStemmer
+# import nltk
+# from nltk.corpus import stopwords
+# from nltk.tokenize import word_tokenize
+# from nltk.stem import PorterStemmer
+#import spacy
 import numpy as np
 import random
 import string
 from typing import List
+from gradio import Progress
 
-nltk.download('punkt')
-nltk.download('stopwords')
-nltk.download('punkt_tab')
+import en_core_web_lg #en_core_web_sm
+nlp = en_core_web_lg.load()
+#from tqdm import tqdm
 
-similarity_threshold = 0.9
+# nltk.download('punkt')
+# nltk.download('stopwords')
+# nltk.download('punkt_tab')
 
-stop_words = set(stopwords.words('english'))
-# List of words to remove from the stopword set
-#words_to_remove = ['no', 'nor', 'not', 'don', 'don't', 'wasn', 'wasn't', 'weren', 'weren't', "don't", "wasn't", "weren't"]
+similarity_threshold = 0.9
 
-# Remove the specified words from the stopwords set
-#for word in words_to_remove:
-#    stop_words.discard(word.lower())
-    
-stemmer = PorterStemmer()
-vectorizer = TfidfVectorizer()
 
-def combine_ocr_output_text(input_files):
+def combine_ocr_output_text(input_files:List[str], output_folder:str=OUTPUT_FOLDER):
     """
     Combines text from multiple CSV files containing page and text columns.
     Groups text by file and page number, concatenating text within these groups.
@@ -92,7 +88,7 @@ def combine_ocr_output_text(input_files):
     
     return combined_df, output_files
 
-def process_data(df, column:str):
+def process_data(df:pd.DataFrame, column:str):
     '''
     Clean and stem text columns in a data frame
     '''
@@ -100,118 +96,130 @@ def process_data(df, column:str):
     def _clean_text(raw_text):
         # Remove HTML tags
         clean = re.sub(r'<.*?>', '', raw_text)
-        clean = re.sub(r'&nbsp;', ' ', clean)
-        clean = re.sub(r'\r\n', ' ', clean)
-        clean = re.sub(r'&lt;', ' ', clean)
-        clean = re.sub(r'&gt;', ' ', clean)
-        clean = re.sub(r'<strong>', ' ', clean)
-        clean = re.sub(r'</strong>', ' ', clean)
+        # clean = re.sub(r'&nbsp;', ' ', clean)
+        # clean = re.sub(r'\r\n', ' ', clean)
+        # clean = re.sub(r'&lt;', ' ', clean)
+        # clean = re.sub(r'&gt;', ' ', clean)
+        # clean = re.sub(r'<strong>', ' ', clean)
+        # clean = re.sub(r'</strong>', ' ', clean)
 
         # Replace non-breaking space \xa0 with a space
-        clean = clean.replace(u'\xa0', u' ')
+        # clean = clean.replace(u'\xa0', u' ')
         # Remove extra whitespace
         clean = ' '.join(clean.split())
 
-        # Tokenize the text
-        words = word_tokenize(clean.lower())
+        # # Tokenize the text
+        # words = word_tokenize(clean.lower())
 
-        # Remove punctuation and numbers
-        words = [word for word in words if word.isalpha()]
+        # # Remove punctuation and numbers
+        # words = [word for word in words if word.isalpha()]
 
-        # Remove stopwords
-        words = [word for word in words if word not in stop_words]
+        # # Remove stopwords
+        # words = [word for word in words if word not in stop_words]
 
         # Join the cleaned words back into a string
-        return ' '.join(words)
-
-    # Function to apply stemming
-    def _apply_stemming(text):
-        # Tokenize the text
-        words = word_tokenize(text.lower())
-        
-        # Apply stemming to each word
-        stemmed_words = [stemmer.stem(word) for word in words]
-        
-        # Join the stemmed words back into a single string
-        return ' '.join(stemmed_words)
-
-
-
-
+        return clean
+
+    # Function to apply lemmatization and remove stopwords
+    def _apply_lemmatization(text):
+        doc = nlp(text)
+        # Keep only alphabetic tokens and remove stopwords
+        lemmatized_words = [token.lemma_ for token in doc if token.is_alpha and not token.is_stop]
+        return ' '.join(lemmatized_words)
+    
     df['text_clean'] = df[column].apply(_clean_text)
-    df['text_clean'] = df['text_clean'].apply(_apply_stemming)
+
+    df['text_clean'] = df['text_clean'].apply(_apply_lemmatization)
     
     return df
 
-def identify_similar_pages(input_files:List[str]):
-
+def identify_similar_pages(input_files: List[str], similarity_threshold: float = 0.9, output_folder:str=OUTPUT_FOLDER, progress=Progress(track_tqdm=True)):
     output_paths = []
+    
+    progress(0.1, desc="Cleaning input texts")
 
+    # Load and clean data
     df, output_files = combine_ocr_output_text(input_files)
-
     output_paths.extend(output_files)
+    df = process_data(df, 'text')  # Assume this returns 'text_clean', 'file', and 'page' columns
 
-    # Clean text
-    df = process_data(df, 'text')
-
-    # Vectorise text
+    # Vectorize text
+    vectorizer = TfidfVectorizer()
     tfidf_matrix = vectorizer.fit_transform(df['text_clean'])
 
-    # Calculate cosine similarity
-    similarity_matrix = cosine_similarity(tfidf_matrix)
+    progress(0.3, desc="Calculating text similarity")
 
-    # Find the indices of the most similar pages
-    np.fill_diagonal(similarity_matrix, 0)  # Ignore self-comparisons
-    similar_pages = np.argwhere(similarity_matrix > similarity_threshold)  # Threshold of similarity
+    # Compute sparse cosine similarity
+    similarity_matrix = cosine_similarity(tfidf_matrix, dense_output=False)  # Keep sparse format
 
-    #print(similar_pages)
+    # Extract indices of similar pages above threshold
+    coo_matrix = similarity_matrix.tocoo()
+    similar_pages = np.array([(i, j, v) for i, j, v in zip(coo_matrix.row, coo_matrix.col, coo_matrix.data) if v > similarity_threshold])
 
-    # Create a DataFrame for similar pairs and their scores
-    similarity_df = pd.DataFrame({
-        'Page1_Index': similar_pages[:, 0],
-        'Page2_Index': similar_pages[:, 1],
-        'Page1_File': similar_pages[:, 0],
-        'Page2_File': similar_pages[:, 1],
-        'Similarity_Score': similarity_matrix[similar_pages[:, 0], similar_pages[:, 1]]
-    })
+    if similar_pages.size == 0:
+        return pd.DataFrame(), output_paths  # Return empty if no matches
+    
+    
 
-    # Filter out duplicate pairs (keep only one direction)
+    # Create a DataFrame for similar pairs
+    similarity_df = pd.DataFrame(similar_pages, columns=['Page1_Index', 'Page2_Index', 'Similarity_Score'])
+    
+    # Remove duplicate pairs (keep one direction)
     similarity_df = similarity_df[similarity_df['Page1_Index'] < similarity_df['Page2_Index']]
 
-    # Map the indices to their corresponding text and metadata
-    similarity_df['Page1_File'] = similarity_df['Page1_File'].map(df['file'])
-    similarity_df['Page2_File'] = similarity_df['Page2_File'].map(df['file'])
+    progress(0.8, desc="Mapping back results")
+    # Map indices to metadata
+    # index_map = df[['file', 'page', 'text']].to_dict(orient='index')
+    # similarity_df['Page1_File'] = similarity_df['Page1_Index'].map(lambda x: index_map[x]['file'])
+    # similarity_df['Page2_File'] = similarity_df['Page2_Index'].map(lambda x: index_map[x]['file'])
+    # similarity_df['Page1_Page'] = similarity_df['Page1_Index'].map(lambda x: index_map[x]['page'])
+    # similarity_df['Page2_Page'] = similarity_df['Page2_Index'].map(lambda x: index_map[x]['page'])
+    # similarity_df['Page1_Text'] = similarity_df['Page1_Index'].map(lambda x: index_map[x]['text'][0:200])
+    # similarity_df['Page2_Text'] = similarity_df['Page2_Index'].map(lambda x: index_map[x]['text'][0:200])
 
-    similarity_df['Page1_Page'] = similarity_df['Page1_Index'].map(df['page'])
-    similarity_df['Page2_Page'] = similarity_df['Page2_Index'].map(df['page'])
+    # Create a DataFrame with the metadata
+    metadata_df = df[['file', 'page', 'text']].reset_index()
 
-    similarity_df['Page1_Text'] = similarity_df['Page1_Index'].map(df['text'])
-    similarity_df['Page2_Text'] = similarity_df['Page2_Index'].map(df['text'])
+    # Merge to get the metadata for Page1
+    similarity_df = similarity_df.merge(metadata_df, left_on='Page1_Index', right_on='index', suffixes=('', '_Page1'))
+    similarity_df = similarity_df.rename(columns={'file': 'Page1_File', 'page': 'Page1_Page', 'text': 'Page1_Text'})
 
+    # Merge to get the metadata for Page2
+    similarity_df = similarity_df.merge(metadata_df, left_on='Page2_Index', right_on='index', suffixes=('', '_Page2'))
+    similarity_df = similarity_df.rename(columns={'file': 'Page2_File', 'page': 'Page2_Page', 'text': 'Page2_Text'})
+
+    # Optionally, drop the index columns if not needed
+    #similarity_df = similarity_df.drop(columns=['index_Page1', 'index_Page2'])
+
+
+    similarity_df["Similarity_Score"] = similarity_df["Similarity_Score"].round(3)
+
+    # Sort results
     similarity_df_out = similarity_df[['Page1_File', 'Page1_Page', 'Page2_File', 'Page2_Page', 'Similarity_Score', 'Page1_Text', 'Page2_Text']]
     similarity_df_out = similarity_df_out.sort_values(["Page1_File", "Page1_Page", "Page2_File", "Page2_Page", "Similarity_Score"], ascending=[True, True, True, True, False])
 
-    # Save detailed results to a CSV file
+    similarity_df_out['Page1_Text'] = similarity_df_out['Page1_Text'][0:100]
+    similarity_df_out['Page2_Text'] = similarity_df_out['Page2_Text'][0:100]
+
+    progress(0.8, desc="Saving output files")
+
+    # Save results
     similarity_file_output_path = output_folder + 'page_similarity_results.csv'
     similarity_df_out.to_csv(similarity_file_output_path, index=False)
-
     output_paths.append(similarity_file_output_path)
 
-    if not similarity_df_out.empty:
-        unique_files = similarity_df_out['Page2_File'].unique()
-        for redact_file in unique_files:
-            output_file_name = output_folder + redact_file + "_whole_page.csv"
-            whole_pages_to_redact_df = similarity_df_out.loc[similarity_df_out['Page2_File']==redact_file,:][['Page2_Page']]
-            whole_pages_to_redact_df.to_csv(output_file_name, header=None, index=None)
-
-            output_paths.append(output_file_name)            
-
+    # Save per-file redaction lists
+    for redact_file in similarity_df_out['Page2_File'].unique():
+        output_file_name = output_folder + redact_file + "_whole_page.csv"
+        whole_pages_to_redact_df = similarity_df_out.loc[similarity_df_out['Page2_File'] == redact_file, ['Page2_Page']].drop_duplicates(['Page2_Page']).sort_values('Page2_Page')
+        whole_pages_to_redact_df.to_csv(output_file_name, header=False, index=False)
+        output_paths.append(output_file_name)
 
     return similarity_df_out, output_paths
 
 # Perturb text
 # Apply the perturbation function with a 10% error probability
-def perturb_text_with_errors(series):
+def perturb_text_with_errors(series:pd.Series):
 
     def _perturb_text(text, error_probability=0.1):
         words = text.split()  # Split text into words
@@ -241,36 +249,3 @@ def perturb_text_with_errors(series):
     series = series.apply(lambda x: _perturb_text(x, error_probability=0.1))
 
     return series
-
-# Run through command line
-# def main():
-#     parser = argparse.ArgumentParser(description='Combine text from multiple CSV files by page')
-#     parser.add_argument('input_pattern', help='Input file pattern (e.g., "input/*.csv")')
-#     parser.add_argument('--output', '-o', default='combined_text.csv', 
-#                        help='Output CSV file path (default: combined_text.csv)')
-
-#     args = parser.parse_args()
-    
-#     # Get list of input files
-#     input_files = glob.glob(args.input_pattern)
-    
-#     if not input_files:
-#         print(f"No files found matching pattern: {args.input_pattern}")
-#         return
-    
-#     print(f"Processing {len(input_files)} files...")
-    
-#     try:
-#         # Combine the text from all files
-#         combined_df = combine_ocr_output_text(input_files)
-        
-#         # Save to CSV
-#         combined_df.to_csv(args.output, index=False)
-#         print(f"Successfully created combined output: {args.output}")
-#         print(f"Total pages processed: {len(combined_df)}")
-        
-#     except Exception as e:
-#         print(f"Error processing files: {str(e)}")
-
-# if __name__ == "__main__":
-#     main()

tools/helper_functions.py

@@ -7,20 +7,21 @@ import pandas as pd
 import numpy as np
 import unicodedata
 from typing import List
+from math import ceil
 from gradio_image_annotation import image_annotator
-from tools.auth import user_pool_id
-from tools.config import CUSTOM_HEADER_VALUE, CUSTOM_HEADER, output_folder, session_output_folder
+from tools.config import CUSTOM_HEADER_VALUE, CUSTOM_HEADER, OUTPUT_FOLDER, INPUT_FOLDER, SESSION_OUTPUT_FOLDER, AWS_USER_POOL_ID
 
 # Names for options labels
 text_ocr_option = "Local model - selectable text"
 tesseract_ocr_option = "Local OCR model - PDFs without selectable text"
 textract_option = "AWS Textract service - all PDF types"
 
+no_redaction_option = "Only extract text (no redaction)"
 local_pii_detector = "Local"
 aws_pii_detector  = "AWS Comprehend"
 
 def reset_state_vars():
-    return [], [], pd.DataFrame(), pd.DataFrame(), 0, "", image_annotator(
+    return [], pd.DataFrame(), pd.DataFrame(), 0, "", image_annotator(
             label="Modify redaction boxes",
             label_list=["Redaction"],
             label_colors=[(0, 0, 0)],
@@ -30,7 +31,10 @@ def reset_state_vars():
             show_share_button=False,
             show_remove_button=False,
             interactive=False
-        ), [], [], pd.DataFrame(), pd.DataFrame(), []
+        ), [], [], pd.DataFrame(), pd.DataFrame(), [], [], ""
+
+def reset_ocr_results_state():
+    return pd.DataFrame(), pd.DataFrame(), []
 
 def reset_review_vars():
     return pd.DataFrame(), pd.DataFrame()
@@ -40,6 +44,23 @@ def load_in_default_allow_list(allow_list_file_path):
         allow_list_file_path = [allow_list_file_path]
     return allow_list_file_path
 
+def load_in_default_cost_codes(cost_codes_path:str):
+    cost_codes_df = pd.read_csv(cost_codes_path)
+
+    dropdown_choices = cost_codes_df.iloc[:,0].to_list()
+    dropdown_choices.insert(0, "")
+
+
+    out_dropdown = gr.Dropdown(value="", label="Choose cost code for analysis", choices=dropdown_choices, allow_custom_value=True)
+    
+    return cost_codes_df, out_dropdown
+
+def enforce_cost_codes(enforce_cost_code_textbox, cost_code_choice):
+    if enforce_cost_code_textbox == "True":
+        if not cost_code_choice:
+            raise Exception("Please choose a cost code before continuing")
+    return
+
 def update_dataframe(df:pd.DataFrame):
     df_copy = df.copy()
     return df_copy
@@ -87,23 +108,20 @@ def read_file(filename):
     elif file_type == 'parquet':
         return pd.read_parquet(filename)
 
-def ensure_output_folder_exists():
-    """Checks if the 'output/' folder exists, creates it if not."""
+def ensure_output_folder_exists(output_folder:str):
+    """Checks if the specified folder exists, creates it if not."""   
 
-    folder_name = "output/"
-
-    if not os.path.exists(folder_name):
+    if not os.path.exists(output_folder):
         # Create the folder if it doesn't exist
-        os.makedirs(folder_name)
-        print(f"Created the 'output/' folder.")
+        os.makedirs(output_folder)
+        print(f"Created the {output_folder} folder.")
     else:
-        print(f"The 'output/' folder already exists.")
+        print(f"The {output_folder} folder already exists.")
 
 def custom_regex_load(in_file:List[str], file_type:str = "allow_list"):
     '''
     When file is loaded, update the column dropdown choices and write to relevant data states.
     '''
-
     custom_regex_df = pd.DataFrame()
 
     if in_file:
@@ -113,7 +131,6 @@ def custom_regex_load(in_file:List[str], file_type:str = "allow_list"):
         if regex_file_names:
             regex_file_name = regex_file_names[0]
             custom_regex_df = pd.read_csv(regex_file_name, low_memory=False, header=None)
-            #regex_file_name_no_ext = get_file_name_without_type(regex_file_name)
             
             # Select just first columns
             custom_regex_df = pd.DataFrame(custom_regex_df.iloc[:,[0]])
@@ -122,8 +139,6 @@ def custom_regex_load(in_file:List[str], file_type:str = "allow_list"):
             custom_regex_df.columns = custom_regex_df.columns.astype(str)
 
             output_text = file_type + " file loaded."
-
-            print("Custom regex df:", custom_regex_df)
             print(output_text)
     else:
         output_text = "No file provided."
@@ -132,7 +147,7 @@ def custom_regex_load(in_file:List[str], file_type:str = "allow_list"):
        
     return output_text, custom_regex_df
 
-def put_columns_in_df(in_file):
+def put_columns_in_df(in_file:List[str]):
     new_choices = []
     concat_choices = []
     all_sheet_names = []
@@ -176,6 +191,16 @@ def put_columns_in_df(in_file):
     else:
         return gr.Dropdown(choices=concat_choices, value=concat_choices), gr.Dropdown(visible=False)
 
+def check_for_existing_textract_file(doc_file_name_no_extension_textbox:str, output_folder:str=OUTPUT_FOLDER):
+    textract_output_path = os.path.join(output_folder, doc_file_name_no_extension_textbox + "_textract.json")
+
+    if os.path.exists(textract_output_path):
+        print("Existing Textract file found.")    
+        return True
+    
+    else:
+        return False
+
 # Following function is only relevant for locally-created executable files based on this app (when using pyinstaller it creates a _internal folder that contains tesseract and poppler. These need to be added to the system path to enable the app to run)
 def add_folder_to_path(folder_path: str):
     '''
@@ -202,7 +227,7 @@ def add_folder_to_path(folder_path: str):
 def reveal_feedback_buttons():
     return gr.Radio(visible=True, label="Please give some feedback about the results of the redaction. A reminder that the app is only expected to identify about 60% of personally identifiable information in a given (typed) document."), gr.Textbox(visible=True), gr.Button(visible=True), gr.Markdown(visible=True)
 
-def wipe_logs(feedback_logs_loc, usage_logs_loc):
+def wipe_logs(feedback_logs_loc:str, usage_logs_loc:str):
     try:
         os.remove(feedback_logs_loc)
     except Exception as e:
@@ -212,7 +237,7 @@ def wipe_logs(feedback_logs_loc, usage_logs_loc):
     except Exception as e:
         print("Could not remove usage logs file", e)
 
-def merge_csv_files(file_list):
+def merge_csv_files(file_list:List[str], output_folder:str=OUTPUT_FOLDER):
 
     # Initialise an empty list to hold DataFrames
     dataframes = []
@@ -246,9 +271,9 @@ def merge_csv_files(file_list):
 
     return output_files
 
-async def get_connection_params(request: gr.Request, output_folder_textbox:str='/output/'):
+async def get_connection_params(request: gr.Request, output_folder_textbox:str=OUTPUT_FOLDER, input_folder_textbox:str=INPUT_FOLDER, session_output_folder:str=SESSION_OUTPUT_FOLDER):
 
-    print("Session hash:", request.session_hash)
+    #print("Session hash:", request.session_hash)
 
     if CUSTOM_HEADER and CUSTOM_HEADER_VALUE:
             if CUSTOM_HEADER in request.headers:
@@ -266,11 +291,11 @@ async def get_connection_params(request: gr.Request, output_folder_textbox:str='
 
     if request.username:
         out_session_hash = request.username
-        print("Request username found:", out_session_hash)
+        #print("Request username found:", out_session_hash)
 
     elif 'x-cognito-id' in request.headers:
         out_session_hash = request.headers['x-cognito-id']
-        print("Cognito ID found:", out_session_hash)
+        #print("Cognito ID found:", out_session_hash)
 
     elif 'x-amzn-oidc-identity' in request.headers:
         out_session_hash = request.headers['x-amzn-oidc-identity']
@@ -279,7 +304,7 @@ async def get_connection_params(request: gr.Request, output_folder_textbox:str='
         cognito_client = boto3.client('cognito-idp')
         try:
             response = cognito_client.admin_get_user(
-                UserPoolId=user_pool_id,  # Replace with your User Pool ID
+                UserPoolId=AWS_USER_POOL_ID,  # Replace with your User Pool ID
                 Username=out_session_hash
             )
             email = next(attr['Value'] for attr in response['UserAttributes'] if attr['Name'] == 'email')
@@ -297,19 +322,19 @@ async def get_connection_params(request: gr.Request, output_folder_textbox:str='
 
     if session_output_folder == 'True':
         output_folder = output_folder_textbox + out_session_hash + "/"
+        input_folder = input_folder_textbox + out_session_hash + "/"
     else:
         output_folder = output_folder_textbox
+        input_folder = input_folder_textbox
 
-    if not os.path.exists(output_folder):
-        os.mkdir(output_folder)
+    if not os.path.exists(output_folder): os.mkdir(output_folder)
+    if not os.path.exists(input_folder): os.mkdir(input_folder)
 
-    #if bucket_name:
-    #    print("S3 output folder is: " + "s3://" + bucket_name + "/" + output_folder)
 
-    return out_session_hash, output_folder, out_session_hash
+    return out_session_hash, output_folder, out_session_hash, input_folder
 
-def clean_unicode_text(text):
-    # Step 1: Normalize unicode characters to decompose any special forms
+def clean_unicode_text(text:str):
+    # Step 1: Normalise unicode characters to decompose any special forms
     normalized_text = unicodedata.normalize('NFKC', text)
 
     # Step 2: Replace smart quotes and special punctuation with standard ASCII equivalents
@@ -329,7 +354,7 @@ def clean_unicode_text(text):
 
     return cleaned_text
    
-def load_all_output_files(folder_path:str=output_folder) -> List[str]:
+def load_all_output_files(folder_path:str=OUTPUT_FOLDER) -> List[str]:
     """Get the file paths of all files in the given folder."""
     file_paths = []
     
@@ -343,4 +368,121 @@ def load_all_output_files(folder_path:str=output_folder) -> List[str]:
     
     return file_paths
 
+def calculate_aws_costs(number_of_pages:str,
+                        text_extract_method_radio:str,
+                        handwrite_signature_checkbox:List[str],
+                        pii_identification_method:str,
+                        textract_output_found_checkbox:bool,
+                        only_extract_text_radio:bool,
+                        textract_page_cost:float=1.5/1000,
+                        textract_signature_cost:float=2.0/1000,
+                        comprehend_unit_cost:float=0.0001,
+                        comprehend_size_unit_average:float=250,
+                        average_characters_per_page:float=2000,
+                        textract_option:str=textract_option,
+                        no_redaction_option:str=no_redaction_option,
+                        aws_pii_detector:str=aws_pii_detector):
+    '''
+    Calculate the approximate cost of submitting a document to AWS Textract and/or AWS Comprehend, assuming that Textract outputs do not already exist in the output folder.
+
+    - number_of_pages: The number of pages in the uploaded document(s).
+    - text_extract_method_radio: The method of text extraction.
+    - handwrite_signature_checkbox: Whether signatures are being extracted or not.
+    - pii_identification_method_drop: The method of personally-identifiable information removal.
+    - textract_output_found_checkbox: Whether existing Textract results have been found in the output folder. Assumes that results exist for all pages and files in the output folder.
+    - only_extract_text_radio (bool, optional): Option to only extract text from the document rather than redact.
+    - textract_page_cost (float, optional): AWS pricing for Textract text extraction per page ($).
+    - textract_signature_cost (float, optional): Additional AWS cost above standard AWS Textract extraction for extracting signatures.
+    - comprehend_unit_cost (float, optional): Cost per 'unit' (300 character minimum) for identifying PII in text with AWS Comprehend.
+    - comprehend_size_unit_average (float, optional): Average size of a 'unit' of text passed to AWS Comprehend by the app through the batching process
+    - average_characters_per_page (float, optional): Average number of characters on an A4 page.
+    - textract_option (str, optional): String label for the text_extract_method_radio button for AWS Textract.
+    - no_redaction_option (str, optional): String label for pii_identification_method_drop for no redaction.
+    - aws_pii_detector (str, optional): String label for pii_identification_method_drop for AWS Comprehend.
+    '''
+    text_extraction_cost = 0
+    pii_identification_cost = 0
+    calculated_aws_cost = 0
+    number_of_pages = int(number_of_pages)
+    
+    if textract_output_found_checkbox != True:
+        if text_extract_method_radio == textract_option:
+            text_extraction_cost = number_of_pages * textract_page_cost
+
+            if "Extract signatures" in handwrite_signature_checkbox:
+                text_extraction_cost += (textract_signature_cost * number_of_pages)
+
+    if pii_identification_method != no_redaction_option:
+        if pii_identification_method == aws_pii_detector:
+            comprehend_page_cost = ceil(average_characters_per_page / comprehend_size_unit_average) * comprehend_unit_cost
+            pii_identification_cost = comprehend_page_cost * number_of_pages
+
+    calculated_aws_cost = calculated_aws_cost + text_extraction_cost + pii_identification_cost
+
+    return calculated_aws_cost
+
+def calculate_time_taken(number_of_pages:str,
+                        text_extract_method_radio:str,
+                        pii_identification_method:str,
+                        textract_output_found_checkbox:bool,
+                        only_extract_text_radio:bool,
+                        convert_page_time:float=0.5,
+                        textract_page_time:float=1,
+                        comprehend_page_time:float=1,
+                        local_text_extraction_page_time:float=0.3,
+                        local_pii_redaction_page_time:float=0.5,                        
+                        local_ocr_extraction_page_time:float=1.5,
+                        textract_option:str=textract_option,
+                        text_ocr_option:str=text_ocr_option,
+                        local_ocr_option:str=tesseract_ocr_option,
+                        no_redaction_option:str=no_redaction_option,
+                        aws_pii_detector:str=aws_pii_detector):
+    '''
+    Calculate the approximate time to redact a document.
+
+    - number_of_pages: The number of pages in the uploaded document(s).
+    - text_extract_method_radio: The method of text extraction.
+    - pii_identification_method_drop: The method of personally-identifiable information removal.
+    - only_extract_text_radio (bool, optional): Option to only extract text from the document rather than redact.
+    - textract_page_time (float, optional): Approximate time to query AWS Textract.
+    - comprehend_page_time (float, optional): Approximate time to query text on a page with AWS Comprehend.
+    - local_text_redaction_page_time (float, optional): Approximate time to extract text on a page with the local text redaction option.
+    - local_pii_redaction_page_time (float, optional): Approximate time to redact text on a page with the local text redaction option.
+    - local_ocr_extraction_page_time (float, optional): Approximate time to extract text from a page with the local OCR redaction option.
+    - textract_option (str, optional): String label for the text_extract_method_radio button for AWS Textract.
+    - text_ocr_option (str, optional): String label for text_extract_method_radio for text extraction.
+    - local_ocr_option (str, optional): String label for text_extract_method_radio for local OCR.
+    - no_redaction_option (str, optional): String label for pii_identification_method_drop for no redaction.    
+    - aws_pii_detector (str, optional): String label for pii_identification_method_drop for AWS Comprehend.
+    '''
+    calculated_time_taken = 0
+    page_conversion_time_taken = 0
+    page_extraction_time_taken = 0
+    page_redaction_time_taken = 0
+
+    number_of_pages = int(number_of_pages)
+
+    # Page preparation/conversion to image time
+    if (text_extract_method_radio != text_ocr_option) and (textract_output_found_checkbox != True):
+        page_conversion_time_taken = number_of_pages * convert_page_time
+
+    # Page text extraction time
+    if text_extract_method_radio == textract_option:
+        if textract_output_found_checkbox != True:
+            page_extraction_time_taken = number_of_pages * textract_page_time
+    elif text_extract_method_radio == local_ocr_option:
+        page_extraction_time_taken = number_of_pages * local_ocr_extraction_page_time
+    elif text_extract_method_radio == text_ocr_option:
+        page_conversion_time_taken = number_of_pages * local_text_extraction_page_time
+
+    # Page redaction time
+    if pii_identification_method != no_redaction_option:
+        if pii_identification_method == aws_pii_detector:
+            page_redaction_time_taken = number_of_pages * comprehend_page_time
+        else:
+            page_redaction_time_taken = number_of_pages * local_pii_redaction_page_time
+
+    calculated_time_taken = (page_conversion_time_taken + page_extraction_time_taken + page_redaction_time_taken)/60
+
+    return calculated_time_taken
     

tools/load_spacy_model_custom_recognisers.py

@@ -1,4 +1,3 @@
-# %%
 from typing import List
 from presidio_analyzer import AnalyzerEngine, PatternRecognizer, EntityRecognizer, Pattern, RecognizerResult
 from presidio_analyzer.nlp_engine import SpacyNlpEngine, NlpArtifacts

tools/redaction_review.py

@@ -13,10 +13,9 @@ from pymupdf import Document, Rect
 import pymupdf
 #from fitz 
 from PIL import ImageDraw, Image
-from collections import defaultdict
 
-from tools.config import output_folder, CUSTOM_BOX_COLOUR, MAX_IMAGE_PIXELS
-from tools.file_conversion import is_pdf, convert_annotation_json_to_review_df, convert_review_df_to_annotation_json
+from tools.config import OUTPUT_FOLDER, CUSTOM_BOX_COLOUR, MAX_IMAGE_PIXELS, INPUT_FOLDER
+from tools.file_conversion import is_pdf, convert_annotation_json_to_review_df, convert_review_df_to_annotation_json, process_single_page_for_image_conversion, multiply_coordinates_by_page_sizes, convert_annotation_data_to_dataframe, create_annotation_dicts_from_annotation_df, remove_duplicate_images_with_blank_boxes
 from tools.helper_functions import get_file_name_without_type,  detect_file_type
 from tools.file_redaction import redact_page_with_pymupdf
 
@@ -26,21 +25,20 @@ def decrease_page(number:int):
     '''
     Decrease page number for review redactions page.
     '''
-    #print("number:", str(number))
     if number > 1:
         return number - 1, number - 1
     else:
         return 1, 1
 
-def increase_page(number:int, image_annotator_object:AnnotatedImageData):
+def increase_page(number:int, page_image_annotator_object:AnnotatedImageData):
     '''
     Increase page number for review redactions page.
     '''
 
-    if not image_annotator_object:
+    if not page_image_annotator_object:
         return 1, 1
 
-    max_pages = len(image_annotator_object)
+    max_pages = len(page_image_annotator_object)
 
     if number < max_pages:
         return number + 1, number + 1
@@ -57,48 +55,35 @@ def update_zoom(current_zoom_level:int, annotate_current_page:int, decrease:bool
         
     return current_zoom_level, annotate_current_page
 
-def remove_duplicate_images_with_blank_boxes(data: List[dict]) -> List[dict]:
-    '''
-    Remove items from the annotator object where the same page exists twice.
-    '''
-    # Group items by 'image'
-    image_groups = defaultdict(list)
-    for item in data:
-        image_groups[item['image']].append(item)
-
-    # Process each group to prioritize items with non-empty boxes
-    result = []
-    for image, items in image_groups.items():
-        # Filter items with non-empty boxes
-        non_empty_boxes = [item for item in items if item.get('boxes')]
-
-         # Remove 'text' elements from boxes
-        for item in non_empty_boxes:
-            if 'boxes' in item:
-                item['boxes'] = [{k: v for k, v in box.items() if k != 'text'} for box in item['boxes']]
-
-        if non_empty_boxes:
-            # Keep the first entry with non-empty boxes
-            result.append(non_empty_boxes[0])
-        else:
-            # If all items have empty or missing boxes, keep the first item
-            result.append(items[0])
-
-    return result
 
 def update_dropdown_list_based_on_dataframe(df:pd.DataFrame, column:str) -> List["str"]:
     '''
     Gather unique elements from a string pandas Series, then append 'ALL' to the start and return the list.
     '''
+    if isinstance(df, pd.DataFrame):
+        # Check if the Series is empty or all NaN
+        if column not in df.columns or df[column].empty or df[column].isna().all():
+            return ["ALL"]
+        elif column != "page":
+            entities = df[column].astype(str).unique().tolist()        
+            entities_for_drop = sorted(entities)
+            entities_for_drop.insert(0, "ALL")
+        else:
+            # Ensure the column can be converted to int - assumes it is the page column
+            try:
+                entities = df[column].astype(int).unique()
+                entities_for_drop = sorted(entities)
+                entities_for_drop = [str(e) for e in entities_for_drop]  # Convert back to string
+                entities_for_drop.insert(0, "ALL")
+            except ValueError:
+                return ["ALL"]  # Handle case where conversion fails
+
+        return entities_for_drop  # Ensure to return the list
+    else:
+        return ["ALL"]
 
-    entities = df[column].astype(str).unique().tolist()        
-    entities_for_drop = sorted(entities)
-    entities_for_drop.insert(0, "ALL")
-
-    return entities_for_drop
-
-def get_filtered_recogniser_dataframe_and_dropdowns(image_annotator_object:AnnotatedImageData,
-                                 recogniser_dataframe_modified:pd.DataFrame,
+def get_filtered_recogniser_dataframe_and_dropdowns(page_image_annotator_object:AnnotatedImageData,
+                                 recogniser_dataframe_base:pd.DataFrame,
                                  recogniser_dropdown_value:str,
                                  text_dropdown_value:str,
                                  page_dropdown_value:str,
@@ -109,10 +94,12 @@ def get_filtered_recogniser_dataframe_and_dropdowns(image_annotator_object:Annot
     '''
 
     recogniser_entities_list = ["Redaction"]
-    recogniser_dataframe_out = recogniser_dataframe_modified
+    recogniser_dataframe_out = recogniser_dataframe_base
+    recogniser_dataframe_out_gr = gr.Dataframe()
+    review_dataframe = review_df
 
     try:
-        review_dataframe = convert_annotation_json_to_review_df(image_annotator_object, review_df, page_sizes)
+        review_dataframe = convert_annotation_json_to_review_df(page_image_annotator_object, review_df, page_sizes)
 
         recogniser_entities_for_drop = update_dropdown_list_based_on_dataframe(review_dataframe, "label")
         recogniser_entities_drop = gr.Dropdown(value=recogniser_dropdown_value, choices=recogniser_entities_for_drop, allow_custom_value=True, interactive=True)
@@ -127,49 +114,96 @@ def get_filtered_recogniser_dataframe_and_dropdowns(image_annotator_object:Annot
         page_entities_for_drop = update_dropdown_list_based_on_dataframe(review_dataframe, "page")
         page_entities_drop = gr.Dropdown(value=page_dropdown_value, choices=page_entities_for_drop, allow_custom_value=True, interactive=True)
 
-        recogniser_dataframe_out = gr.Dataframe(review_dataframe[["page", "label", "text"]], show_search="filter", col_count=(3, "fixed"), type="pandas", headers=["page", "label", "text"])
+        recogniser_dataframe_out_gr = gr.Dataframe(review_dataframe[["page", "label", "text"]], show_search="filter", col_count=(3, "fixed"), type="pandas", headers=["page", "label", "text"], show_fullscreen_button=True, wrap=True)
+
+        recogniser_dataframe_out = review_dataframe[["page", "label", "text"]]
 
     except Exception as e:
         print("Could not extract recogniser information:", e)
-        recogniser_dataframe_out = recogniser_dataframe_modified[["page", "label", "text"]]
+        recogniser_dataframe_out = recogniser_dataframe_base[["page", "label", "text"]]
 
-        recogniser_entities_drop = gr.Dropdown(value=recogniser_dropdown_value, choices=recogniser_dataframe_out["label"].astype(str).unique().tolist(), allow_custom_value=True, interactive=True)
+        label_choices = review_dataframe["label"].astype(str).unique().tolist()
+        text_choices = review_dataframe["text"].astype(str).unique().tolist()
+        page_choices = review_dataframe["page"].astype(str).unique().tolist()
+
+        recogniser_entities_drop = gr.Dropdown(value=recogniser_dropdown_value, choices=label_choices, allow_custom_value=True, interactive=True)
         recogniser_entities_list = ["Redaction"]
-        text_entities_drop = gr.Dropdown(value=text_dropdown_value, choices=recogniser_dataframe_out["text"].astype(str).unique().tolist(), allow_custom_value=True, interactive=True)
-        page_entities_drop = gr.Dropdown(value=page_dropdown_value, choices=recogniser_dataframe_out["page"].astype(str).unique().tolist(), allow_custom_value=True, interactive=True)
+        text_entities_drop = gr.Dropdown(value=text_dropdown_value, choices=text_choices, allow_custom_value=True, interactive=True)
+        page_entities_drop = gr.Dropdown(value=page_dropdown_value, choices=page_choices, allow_custom_value=True, interactive=True)
 
-    return recogniser_dataframe_out, recogniser_dataframe_out, recogniser_entities_drop, recogniser_entities_list, text_entities_drop, page_entities_drop
+    return recogniser_dataframe_out_gr, recogniser_dataframe_out, recogniser_entities_drop, recogniser_entities_list, text_entities_drop, page_entities_drop
 
-def update_recogniser_dataframes(image_annotator_object:AnnotatedImageData, recogniser_dataframe_modified:pd.DataFrame, recogniser_entities_dropdown_value:str="ALL", text_dropdown_value:str="ALL", page_dropdown_value:str="ALL", review_df:pd.DataFrame=[], page_sizes:list[str]=[]):
+def update_recogniser_dataframes(page_image_annotator_object:AnnotatedImageData, recogniser_dataframe_base:pd.DataFrame, recogniser_entities_dropdown_value:str="ALL", text_dropdown_value:str="ALL", page_dropdown_value:str="ALL", review_df:pd.DataFrame=[], page_sizes:list[str]=[]):
     '''
     Update recogniser dataframe information that appears alongside the pdf pages on the review screen.
     '''
     recogniser_entities_list = ["Redaction"]
     recogniser_dataframe_out = pd.DataFrame()
+    recogniser_dataframe_out_gr = gr.Dataframe()
+
+    # If base recogniser dataframe is empy, need to create it.
+    if recogniser_dataframe_base.empty:
+        recogniser_dataframe_out_gr, recogniser_dataframe_out, recogniser_entities_drop, recogniser_entities_list, text_entities_drop, page_entities_drop = get_filtered_recogniser_dataframe_and_dropdowns(page_image_annotator_object, recogniser_dataframe_base, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df, page_sizes)    
+    elif recogniser_dataframe_base.iloc[0,0] == "":
+        recogniser_dataframe_out_gr, recogniser_dataframe_out, recogniser_entities_dropdown_value, recogniser_entities_list, text_entities_drop, page_entities_drop = get_filtered_recogniser_dataframe_and_dropdowns(page_image_annotator_object, recogniser_dataframe_base, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df, page_sizes)
+    else:
+        recogniser_dataframe_out_gr, recogniser_dataframe_out, recogniser_entities_dropdown, recogniser_entities_list, text_dropdown, page_dropdown = get_filtered_recogniser_dataframe_and_dropdowns(page_image_annotator_object, recogniser_dataframe_base, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df, page_sizes)
+
+        review_dataframe, text_entities_drop, page_entities_drop = update_entities_df_recogniser_entities(recogniser_entities_dropdown_value, recogniser_dataframe_out, page_dropdown_value, text_dropdown_value)
 
-    if recogniser_dataframe_modified.empty:
-        recogniser_dataframe_modified, recogniser_dataframe_out, recogniser_entities_drop, recogniser_entities_list, text_entities_drop, page_entities_drop = get_filtered_recogniser_dataframe_and_dropdowns(image_annotator_object, recogniser_dataframe_modified, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df, page_sizes)    
-    elif recogniser_dataframe_modified.iloc[0,0] == "":
-        recogniser_dataframe_modified, recogniser_dataframe_out, recogniser_entities_dropdown_value, recogniser_entities_list, text_entities_drop, page_entities_drop = get_filtered_recogniser_dataframe_and_dropdowns(image_annotator_object, recogniser_dataframe_modified, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df, page_sizes)
-    else:        
-        review_dataframe, text_entities_drop, page_entities_drop = update_entities_df_recogniser_entities(recogniser_entities_dropdown_value, recogniser_dataframe_modified, page_dropdown_value, text_dropdown_value)
-        recogniser_dataframe_out = gr.Dataframe(review_dataframe[["page", "label", "text"]], show_search="filter", col_count=(3, "fixed"), type="pandas", headers=["page", "label", "text"])
+        recogniser_dataframe_out_gr = gr.Dataframe(review_dataframe[["page", "label", "text"]], show_search="filter", col_count=(3, "fixed"), type="pandas", headers=["page", "label", "text"], show_fullscreen_button=True, wrap=True)
         
-        recogniser_entities_for_drop = update_dropdown_list_based_on_dataframe(recogniser_dataframe_modified, "label")
+        recogniser_entities_for_drop = update_dropdown_list_based_on_dataframe(recogniser_dataframe_out, "label")
         recogniser_entities_drop = gr.Dropdown(value=recogniser_entities_dropdown_value, choices=recogniser_entities_for_drop, allow_custom_value=True, interactive=True)
 
-        recogniser_entities_list_base = recogniser_dataframe_modified["label"].astype(str).unique().tolist()
+        recogniser_entities_list_base = recogniser_dataframe_out["label"].astype(str).unique().tolist()
 
         # Recogniser entities list is the list of choices that appear when you make a new redaction box
         recogniser_entities_list = [entity for entity in recogniser_entities_list_base if entity != 'Redaction']
         recogniser_entities_list.insert(0, 'Redaction')
 
-    return recogniser_entities_list, recogniser_dataframe_out, recogniser_dataframe_modified, recogniser_entities_drop, text_entities_drop, page_entities_drop
+    return recogniser_entities_list, recogniser_dataframe_out_gr, recogniser_dataframe_out, recogniser_entities_drop, text_entities_drop, page_entities_drop
 
 def undo_last_removal(backup_review_state, backup_image_annotations_state, backup_recogniser_entity_dataframe_base):
     return backup_review_state, backup_image_annotations_state, backup_recogniser_entity_dataframe_base
 
-def exclude_selected_items_from_redaction(review_df: pd.DataFrame, selected_rows_df: pd.DataFrame, image_file_paths:List[str], page_sizes:List[dict], image_annotations_state:dict, recogniser_entity_dataframe_base:pd.DataFrame):
+def update_annotator_page_from_review_df(review_df: pd.DataFrame,
+                                          image_file_paths:List[str],
+                                          page_sizes:List[dict],
+                                          current_page:int,
+                                          previous_page:int,
+                                          current_image_annotations_state:List[str],
+                                          current_page_annotator:object):
+    '''
+    Update the visible annotation object with the latest review file information
+    '''
+    out_image_annotations_state = current_image_annotations_state
+    out_current_page_annotator = current_page_annotator
+
+    print("page_sizes:", page_sizes)
+
+    review_df.to_csv(OUTPUT_FOLDER + "review_df_in_update_annotator.csv")
+
+    if not review_df.empty:
+
+        out_image_annotations_state = convert_review_df_to_annotation_json(review_df, image_file_paths, page_sizes)
+
+        print("out_image_annotations_state[current_page-1]:", out_image_annotations_state[current_page-1])
+
+        if previous_page == current_page:
+            out_current_page_annotator = out_image_annotations_state[current_page-1]
+
+    return out_current_page_annotator, out_image_annotations_state
+
+
+
+
+def exclude_selected_items_from_redaction(review_df: pd.DataFrame,
+                                          selected_rows_df: pd.DataFrame,
+                                          image_file_paths:List[str],
+                                          page_sizes:List[dict],
+                                          image_annotations_state:dict,
+                                          recogniser_entity_dataframe_base:pd.DataFrame):
     '''
     Remove selected items from the review dataframe from the annotation object and review dataframe.
     '''
@@ -180,7 +214,7 @@ def exclude_selected_items_from_redaction(review_df: pd.DataFrame, selected_rows
 
     if not selected_rows_df.empty and not review_df.empty:
         # Ensure selected_rows_df has the same relevant columns
-        selected_subset = selected_rows_df[['label', 'page', 'text']].drop_duplicates()
+        selected_subset = selected_rows_df[['label', 'page', 'text']].drop_duplicates(subset=['label', 'page', 'text'])
 
         # Perform anti-join using merge with an indicator column
         merged_df = review_df.merge(selected_subset, on=['label', 'page', 'text'], how='left', indicator=True)
@@ -189,6 +223,7 @@ def exclude_selected_items_from_redaction(review_df: pd.DataFrame, selected_rows
         out_review_df = merged_df[merged_df['_merge'] == 'left_only'].drop(columns=['_merge'])
 
         out_image_annotations_state = convert_review_df_to_annotation_json(out_review_df, image_file_paths, page_sizes)
+
         out_recogniser_entity_dataframe_base = out_review_df[["page", "label", "text"]]
     
     # Either there is nothing left in the selection dataframe, or the review dataframe
@@ -196,90 +231,118 @@ def exclude_selected_items_from_redaction(review_df: pd.DataFrame, selected_rows
         out_review_df = review_df
         out_recogniser_entity_dataframe_base = recogniser_entity_dataframe_base
 
-        out_image_annotations_state = []
-
-        for page_no, page in enumerate(image_file_paths):
-            annotation = {}
-            annotation["image"] = image_file_paths[page_no]
-            annotation["boxes"] = []
+        out_image_annotations_state = image_annotations_state
 
-            out_image_annotations_state.append(annotation)
-    
     return out_review_df, out_image_annotations_state, out_recogniser_entity_dataframe_base, backup_review_state, backup_image_annotations_state, backup_recogniser_entity_dataframe_base
 
-def update_annotator(image_annotator_object:AnnotatedImageData,
-                     page_num:int,
-                     recogniser_entities_dropdown_value:str="ALL",
-                     page_dropdown_value:str="ALL",
-                     text_dropdown_value:str="ALL",
-                     recogniser_dataframe_modified=gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), type="pandas", headers=["page", "label", "text"]), zoom:int=100,
-                     review_df:pd.DataFrame=[],
-                     page_sizes:List[dict]=[]):
+def update_annotator_object_and_filter_df(
+                    all_image_annotations:List[AnnotatedImageData],
+                    gradio_annotator_current_page_number:int,
+                    recogniser_entities_dropdown_value:str="ALL",
+                    page_dropdown_value:str="ALL",
+                    text_dropdown_value:str="ALL",
+                    recogniser_dataframe_base:gr.Dataframe=gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), type="pandas", headers=["page", "label", "text"], show_fullscreen_button=True, wrap=True),
+                    zoom:int=100,
+                    review_df:pd.DataFrame=[],
+                    page_sizes:List[dict]=[],
+                    doc_full_file_name_textbox:str='',
+                    input_folder:str=INPUT_FOLDER):
     '''
     Update a gradio_image_annotation object with new annotation data.
     '''
-    # First, update the dataframe containing the found recognisers
-    recogniser_entities_list, recogniser_dataframe_out, recogniser_dataframe_modified, recogniser_entities_dropdown_value, text_entities_drop, page_entities_drop = update_recogniser_dataframes(image_annotator_object, recogniser_dataframe_modified, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df, page_sizes)
+    zoom_str = str(zoom) + '%'
+    
+    if not gradio_annotator_current_page_number: gradio_annotator_current_page_number = 0
 
-    #print("Creating output annotator object in update_annotator function")
+    # Check bounding values for current page and page max
+    if gradio_annotator_current_page_number > 0: page_num_reported = gradio_annotator_current_page_number
+    elif gradio_annotator_current_page_number == 0: page_num_reported = 1 # minimum possible reported page is 1
+    else: 
+        gradio_annotator_current_page_number = 0
+        page_num_reported = 1
 
-    zoom_str = str(zoom) + '%'
-    recogniser_colour_list = [(0, 0, 0) for _ in range(len(recogniser_entities_list))]
+    # Ensure page displayed can't exceed number of pages in document
+    page_max_reported = len(all_image_annotations)
+    if page_num_reported > page_max_reported: page_num_reported = page_max_reported
 
-    #print("recogniser_entities_list:", recogniser_entities_list)
-    #print("recogniser_colour_list:", recogniser_colour_list)
-    #print("zoom_str:", zoom_str)
+    page_num_reported_zero_indexed = page_num_reported - 1
 
-    if not image_annotator_object:
-        page_num_reported = 1
+    # First, check that the image on the current page is valid, replace with what exists in page_sizes object if not
+    page_image_annotator_object, all_image_annotations = replace_images_in_image_annotation_object(all_image_annotations, all_image_annotations[page_num_reported_zero_indexed], page_sizes, page_num_reported)    
 
-        out_image_annotator = image_annotator(
-        None,
-        boxes_alpha=0.1,
-        box_thickness=1,
-        label_list=recogniser_entities_list,
-        label_colors=recogniser_colour_list,
-        show_label=False,
-        height=zoom_str,
-        width=zoom_str,
-        box_min_size=1,
-        box_selected_thickness=2,
-        handle_size=4,
-        sources=None,#["upload"],
-        show_clear_button=False,
-        show_share_button=False,
-        show_remove_button=False,
-        handles_cursor=True,
-        interactive=True
-    )        
-        number_reported = gr.Number(label = "Current page", value=page_num_reported, precision=0)
+    all_image_annotations[page_num_reported_zero_indexed] = page_image_annotator_object
+    
+    current_image_path = all_image_annotations[page_num_reported_zero_indexed]['image']
 
-        return out_image_annotator, number_reported, number_reported, page_num_reported, recogniser_entities_dropdown_value, recogniser_dataframe_out, recogniser_dataframe_modified, text_entities_drop, page_entities_drop
+    # If image path is still not valid, load in a new image an overwrite it. Then replace all items in the image annotation object for all pages based on the updated information.
+    page_sizes_df = pd.DataFrame(page_sizes)
+
+    if not os.path.exists(current_image_path):        
+
+        page_num, replaced_image_path, width, height = process_single_page_for_image_conversion(doc_full_file_name_textbox, page_num_reported_zero_indexed, input_folder=input_folder)
+
+        # Overwrite page_sizes values 
+        page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_width"] = width
+        page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_height"] = height
+        page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_path"] = replaced_image_path
     
-    #print("page_num at start of update_annotator function:", page_num)
+    else:
+        if not page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_width"].isnull().all():
+            width = page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_width"].max()
+            height = page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_height"].max()      
+        else:
+            image = Image.open(current_image_path)
+            width = image.width
+            height = image.height
 
-    if page_num is None:
-        page_num = 0
+            page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_width"] = width
+            page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_height"] = height
 
-    # Check bounding values for current page and page max
-    if page_num > 0:
-        page_num_reported = page_num
+        page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_path"] = current_image_path
 
-    elif page_num == 0: page_num_reported = 1
+        replaced_image_path = current_image_path
 
-    else: 
-        page_num = 0   
-        page_num_reported = 1 
+    if review_df.empty: review_df = pd.DataFrame(columns=["image", "page", "label", "color", "xmin", "ymin", "xmax", "ymax", "text"])
+
+    ##
+    
+    review_df.loc[review_df["page"]==page_num_reported, 'image'] = replaced_image_path
 
-    page_max_reported = len(image_annotator_object)
+    # Update dropdowns and review selection dataframe with the updated annotator object
+    recogniser_entities_list, recogniser_dataframe_out_gr, recogniser_dataframe_modified, recogniser_entities_dropdown_value, text_entities_drop, page_entities_drop = update_recogniser_dataframes(all_image_annotations, recogniser_dataframe_base, recogniser_entities_dropdown_value, text_dropdown_value, page_dropdown_value, review_df.copy(), page_sizes)
+    
+    recogniser_colour_list = [(0, 0, 0) for _ in range(len(recogniser_entities_list))]
 
-    if page_num_reported > page_max_reported:
-        page_num_reported = page_max_reported
+    # page_sizes_df has been changed - save back to page_sizes_object
+    page_sizes = page_sizes_df.to_dict(orient='records')
 
-    image_annotator_object = remove_duplicate_images_with_blank_boxes(image_annotator_object)
+    images_list = list(page_sizes_df["image_path"])
+    images_list[page_num_reported_zero_indexed] = replaced_image_path
+
+    all_image_annotations[page_num_reported_zero_indexed]['image'] = replaced_image_path
     
-    out_image_annotator = image_annotator(
-        value = image_annotator_object[page_num_reported - 1],
+    # Multiply out image_annotation coordinates from relative to absolute if necessary
+    all_image_annotations_df = convert_annotation_data_to_dataframe(all_image_annotations)
+
+    all_image_annotations_df = multiply_coordinates_by_page_sizes(all_image_annotations_df, page_sizes_df, xmin="xmin", xmax="xmax", ymin="ymin", ymax="ymax")   
+
+    all_image_annotations = create_annotation_dicts_from_annotation_df(all_image_annotations_df, page_sizes)
+
+    # Remove blank duplicate entries
+    all_image_annotations = remove_duplicate_images_with_blank_boxes(all_image_annotations)
+
+    current_page_image_annotator_object = all_image_annotations[page_num_reported_zero_indexed]
+
+    page_number_reported_gradio = gr.Number(label = "Current page", value=page_num_reported, precision=0)
+    
+    ###
+    # If no data, present a blank page
+    if not all_image_annotations:
+        print("No all_image_annotation object found")
+        page_num_reported = 1
+
+        out_image_annotator = image_annotator(
+        value = None,
         boxes_alpha=0.1,
         box_thickness=1,
         label_list=recogniser_entities_list,
@@ -296,47 +359,99 @@ def update_annotator(image_annotator_object:AnnotatedImageData,
         show_remove_button=False,
         handles_cursor=True,
         interactive=True
-    )
+        )        
 
-    number_reported = gr.Number(label = "Current page", value=page_num_reported, precision=0)
+        return out_image_annotator, page_number_reported_gradio, page_number_reported_gradio, page_num_reported, recogniser_entities_dropdown_value, recogniser_dataframe_out_gr, recogniser_dataframe_modified, text_entities_drop, page_entities_drop, page_sizes, all_image_annotations
+    
+    else:
+        ### Present image_annotator outputs
+        out_image_annotator = image_annotator(
+            value = current_page_image_annotator_object,
+            boxes_alpha=0.1,
+            box_thickness=1,
+            label_list=recogniser_entities_list,
+            label_colors=recogniser_colour_list,
+            show_label=False,
+            height=zoom_str,
+            width=zoom_str,
+            box_min_size=1,
+            box_selected_thickness=2,
+            handle_size=4,
+            sources=None,#["upload"],
+            show_clear_button=False,
+            show_share_button=False,
+            show_remove_button=False,
+            handles_cursor=True,
+            interactive=True
+        )
+
+    #print("all_image_annotations at end of update_annotator...:", all_image_annotations)
+    #print("review_df at end of update_annotator_object:", review_df)
+
+    return out_image_annotator, page_number_reported_gradio, page_number_reported_gradio, page_num_reported, recogniser_entities_dropdown_value, recogniser_dataframe_out_gr, recogniser_dataframe_modified, text_entities_drop, page_entities_drop, page_sizes, all_image_annotations
+
+def replace_images_in_image_annotation_object(
+        all_image_annotations:List[dict],
+        page_image_annotator_object:AnnotatedImageData,
+        page_sizes:List[dict],
+        page:int):
+
+        '''
+        Check if the image value in an AnnotatedImageData dict is a placeholder or np.array. If either of these, replace the value with the file path of the image that is hopefully already loaded into the app related to this page.
+        '''
+
+        page_zero_index = page - 1
+        
+        if isinstance(all_image_annotations[page_zero_index]["image"], np.ndarray) or "placeholder_image" in all_image_annotations[page_zero_index]["image"] or isinstance(page_image_annotator_object['image'], np.ndarray):
+            page_sizes_df = pd.DataFrame(page_sizes)
+            page_sizes_df[["page"]] = page_sizes_df[["page"]].apply(pd.to_numeric, errors="coerce")
+
+            # Check for matching pages
+            matching_paths = page_sizes_df.loc[page_sizes_df['page'] == page, "image_path"].unique()
+
+            if matching_paths.size > 0:
+                image_path = matching_paths[0]
+                page_image_annotator_object['image'] = image_path
+                all_image_annotations[page_zero_index]["image"] = image_path
+            else:
+                print(f"No image path found for page {page}.")
 
-    return out_image_annotator, number_reported, number_reported, page_num_reported, recogniser_entities_dropdown_value, recogniser_dataframe_out, recogniser_dataframe_modified, text_entities_drop, page_entities_drop
+        return page_image_annotator_object, all_image_annotations
 
-def modify_existing_page_redactions(image_annotator_object:AnnotatedImageData,
+def update_all_page_annotation_object_based_on_previous_page(
+                                    page_image_annotator_object:AnnotatedImageData,
                                     current_page:int,
                                     previous_page:int,
                                     all_image_annotations:List[AnnotatedImageData],
-                                    recogniser_entities_dropdown_value="ALL",                                    
-                                    text_dropdown_value="ALL",
-                                    page_dropdown_value="ALL",
-                                    recogniser_dataframe=gr.Dataframe(pd.DataFrame(data={"page":[], "label":[], "text":[]}), show_search="filter", col_count=(3, "fixed"), type="pandas", headers=["page", "label", "text"]),
-                                    review_dataframe:pd.DataFrame=[],
                                     page_sizes:List[dict]=[],
                                     clear_all:bool=False
                                     ):
     '''
-    Overwrite current image annotations with modifications
+    Overwrite image annotations on the page we are moving from with modifications.
     '''
 
-    if not current_page:
-        current_page = 1
+    previous_page_zero_index = previous_page -1
+ 
+    if not current_page: current_page = 1
 
-    image_annotator_object['image'] = all_image_annotations[previous_page - 1]["image"]
+    #print("page_image_annotator_object at start of update_all_page_annotation_object:", page_image_annotator_object)
+      
+    page_image_annotator_object, all_image_annotations = replace_images_in_image_annotation_object(all_image_annotations, page_image_annotator_object, page_sizes, previous_page)
 
-    if clear_all == False:
-        all_image_annotations[previous_page - 1] = image_annotator_object
-    else:
-        all_image_annotations[previous_page - 1]["boxes"] = []
+    #print("page_image_annotator_object after replace_images in update_all_page_annotation_object:", page_image_annotator_object)
+
+    if clear_all == False: all_image_annotations[previous_page_zero_index] = page_image_annotator_object
+    else: all_image_annotations[previous_page_zero_index]["boxes"] = []
 
     return all_image_annotations, current_page, current_page
 
-def apply_redactions(image_annotator_object:AnnotatedImageData,
+def apply_redactions_to_review_df_and_files(page_image_annotator_object:AnnotatedImageData,
                      file_paths:List[str],
                      doc:Document,
                      all_image_annotations:List[AnnotatedImageData],
                      current_page:int,
                      review_file_state:pd.DataFrame,
-                     output_folder:str = output_folder,
+                     output_folder:str = OUTPUT_FOLDER,
                      save_pdf:bool=True,
                      page_sizes:List[dict]=[],
                      progress=gr.Progress(track_tqdm=True)):
@@ -347,22 +462,22 @@ def apply_redactions(image_annotator_object:AnnotatedImageData,
     output_files = []
     output_log_files = []
     pdf_doc = []
+    review_df = review_file_state
 
-    #print("File paths in apply_redactions:", file_paths)
+    page_image_annotator_object = all_image_annotations[current_page - 1]   
 
-    image_annotator_object['image'] = all_image_annotations[current_page - 1]["image"]
+    # This replaces the numpy array image object with the image file path
+    page_image_annotator_object, all_image_annotations = replace_images_in_image_annotation_object(all_image_annotations, page_image_annotator_object, page_sizes, current_page)
+    page_image_annotator_object['image'] = all_image_annotations[current_page - 1]["image"]
 
-    all_image_annotations[current_page - 1] = image_annotator_object
-
-    if not image_annotator_object:
-        print("No image annotations found")
-        return doc, all_image_annotations
+    if not page_image_annotator_object:
+        print("No image annotations object found for page")
+        return doc, all_image_annotations, output_files, output_log_files, review_df
     
     if isinstance(file_paths, str):
         file_paths = [file_paths]
 
     for file_path in file_paths:
-        #print("file_path:", file_path)
         file_name_without_ext = get_file_name_without_type(file_path)
         file_name_with_ext = os.path.basename(file_path)
 
@@ -373,11 +488,9 @@ def apply_redactions(image_annotator_object:AnnotatedImageData,
             if (is_pdf(file_path) == False) & (file_extension not in '.csv'):
                 image = Image.open(file_paths[-1])
 
-                #image = pdf_doc
-
                 draw = ImageDraw.Draw(image)
 
-                for img_annotation_box in image_annotator_object['boxes']:
+                for img_annotation_box in page_image_annotator_object['boxes']:
                     coords = [img_annotation_box["xmin"],
                     img_annotation_box["ymin"],
                     img_annotation_box["xmax"],
@@ -385,6 +498,25 @@ def apply_redactions(image_annotator_object:AnnotatedImageData,
 
                     fill = img_annotation_box["color"]
 
+                    # Ensure fill is a valid RGB tuple
+                    if isinstance(fill, tuple) and len(fill) == 3:
+                        # Check if all elements are integers in the range 0-255
+                        if all(isinstance(c, int) and 0 <= c <= 255 for c in fill):
+                            pass
+                            #print("fill:", fill)
+                        else:
+                            print(f"Invalid color values: {fill}. Defaulting to black.")
+                            fill = (0, 0, 0)  # Default to black if invalid
+                    else:
+                        print(f"Invalid fill format: {fill}. Defaulting to black.")
+                        fill = (0, 0, 0)  # Default to black if not a valid tuple
+
+                        # Ensure the image is in RGB mode
+                    if image.mode not in ("RGB", "RGBA"):
+                        image = image.convert("RGB")
+
+                    draw = ImageDraw.Draw(image)
+
                     draw.rectangle(coords, fill=fill)
                     
                     output_image_path = output_folder + file_name_without_ext + "_redacted.png"
@@ -392,12 +524,10 @@ def apply_redactions(image_annotator_object:AnnotatedImageData,
 
                 output_files.append(output_image_path)
 
-                print("Redactions saved to image file")
-
                 doc = [image]
 
             elif file_extension in '.csv':
-                print("This is a csv")
+                #print("This is a csv")
                 pdf_doc = []
 
             # If working with pdfs
@@ -410,46 +540,38 @@ def apply_redactions(image_annotator_object:AnnotatedImageData,
                 number_of_pages = pdf_doc.page_count
                 original_cropboxes = []
 
-                print("Saving pages to file.")
+                page_sizes_df = pd.DataFrame(page_sizes)
+                page_sizes_df[["page"]] = page_sizes_df[["page"]].apply(pd.to_numeric, errors="coerce")
 
                 for i in progress.tqdm(range(0, number_of_pages), desc="Saving redactions to file", unit = "pages"):
-
-                    #print("Saving page", str(i))
-                    
+           
                     image_loc = all_image_annotations[i]['image']
-                    #print("Image location:", image_loc)
 
                     # Load in image object
                     if isinstance(image_loc, np.ndarray):
                         image = Image.fromarray(image_loc.astype('uint8'))
-                        #all_image_annotations[i]['image'] = image_loc.tolist()
                     elif isinstance(image_loc, Image.Image):
                         image = image_loc
-                        #image_out_folder = output_folder + file_name_without_ext + "_page_" + str(i) + ".png"
-                        #image_loc.save(image_out_folder)
-                        #all_image_annotations[i]['image'] = image_out_folder
                     elif isinstance(image_loc, str):
-                        image = Image.open(image_loc)
-
-                    
-                    #print("all_image_annotations for page:", all_image_annotations[i])
-                    #print("image:", image)
+                        if not os.path.exists(image_loc):
+                            image=page_sizes_df.loc[page_sizes_df['page']==i, "image_path"]
+                        try:
+                            image = Image.open(image_loc)
+                        except Exception as e:
+                            image = None
 
                     pymupdf_page = pdf_doc.load_page(i) #doc.load_page(current_page -1)
                     original_cropboxes.append(pymupdf_page.cropbox.irect)
                     pymupdf_page.set_cropbox = pymupdf_page.mediabox
-                    #print("pymupdf_page:", pymupdf_page)
-                    # print("original_cropboxes:", original_cropboxes)
-
-                    pymupdf_page = redact_page_with_pymupdf(page=pymupdf_page, page_annotations=all_image_annotations[i], image=image, original_cropbox=original_cropboxes[-1])
 
+                    pymupdf_page = redact_page_with_pymupdf(page=pymupdf_page, page_annotations=all_image_annotations[i], image=image, original_cropbox=original_cropboxes[-1], page_sizes_df= page_sizes_df) # image=image,
             else:
                 print("File type not recognised.")
                     
             #try:
             if pdf_doc:
                 out_pdf_file_path = output_folder + file_name_without_ext + "_redacted.pdf"
-                pdf_doc.save(out_pdf_file_path)
+                pdf_doc.save(out_pdf_file_path, garbage=4, deflate=True, clean=True)
                 output_files.append(out_pdf_file_path)
 
             else:
@@ -462,38 +584,60 @@ def apply_redactions(image_annotator_object:AnnotatedImageData,
                 output_files.append(orig_pdf_file_path)
 
         try:
-            #print("Saving annotations to JSON")
-
-            # out_annotation_file_path = output_folder + file_name_with_ext + '_review_file.json'
-            # with open(out_annotation_file_path, 'w') as f:
-            #     json.dump(all_image_annotations, f)
-            # output_log_files.append(out_annotation_file_path)
-
-            #print("Saving annotations to CSV review file")
-            #print("all_image_annotations before conversion in apply redactions:", all_image_annotations)
-            #print("review_file_state before conversion in apply redactions:", review_file_state)
-            #print("page_sizes before conversion in apply redactions:", page_sizes)
-
-            # Convert json to csv and also save this
-            review_df = convert_annotation_json_to_review_df(all_image_annotations, review_file_state, page_sizes=page_sizes)[["image",	"page",	"text",	"label","color", "xmin", "ymin", "xmax",	"ymax"]]
+            review_df = convert_annotation_json_to_review_df(all_image_annotations, review_file_state.copy(), page_sizes=page_sizes)[["image",	"page",	"label","color", "xmin", "ymin", "xmax", "ymax", "text"]]#.drop_duplicates(subset=["image",	"page",	"text",	"label","color", "xmin", "ymin", "xmax", "ymax"])
             out_review_file_file_path = output_folder + file_name_with_ext + '_review_file.csv'
 
-            #print("Saving review file after convert_annotation_json_to_review_df function in apply redactions")
             review_df.to_csv(out_review_file_file_path, index=None)
             output_files.append(out_review_file_file_path)
 
         except Exception as e:
             print("In apply redactions function, could not save annotations to csv file:", e)
 
-    return doc, all_image_annotations, output_files, output_log_files
+    return doc, all_image_annotations, output_files, output_log_files, review_df
 
 def get_boxes_json(annotations:AnnotatedImageData):
     return annotations["boxes"]
 
+def update_all_entity_df_dropdowns(df:pd.DataFrame, label_dropdown_value:str, page_dropdown_value:str, text_dropdown_value:str):
+    '''
+    Update all dropdowns based on rows that exist in a dataframe
+    '''
+
+    if isinstance(label_dropdown_value, str):
+        label_dropdown_value = [label_dropdown_value]
+    if isinstance(page_dropdown_value, str):
+        page_dropdown_value = [page_dropdown_value]
+    if isinstance(text_dropdown_value, str):
+        text_dropdown_value = [text_dropdown_value]
+    
+    filtered_df = df.copy()
+
+    # Apply filtering based on dropdown selections
+    # if not "ALL" in page_dropdown_value:
+    #     filtered_df = filtered_df[filtered_df["page"].astype(str).isin(page_dropdown_value)]
+    
+    # if not "ALL" in text_dropdown_value:
+    #     filtered_df = filtered_df[filtered_df["text"].astype(str).isin(text_dropdown_value)]
+
+    # if not "ALL" in label_dropdown_value:
+    #     filtered_df = filtered_df[filtered_df["label"].astype(str).isin(label_dropdown_value)]
+
+    recogniser_entities_for_drop = update_dropdown_list_based_on_dataframe(filtered_df, "label")
+    recogniser_entities_drop = gr.Dropdown(value=label_dropdown_value[0], choices=recogniser_entities_for_drop, allow_custom_value=True, interactive=True)    
+
+    text_entities_for_drop = update_dropdown_list_based_on_dataframe(filtered_df, "text")
+    text_entities_drop = gr.Dropdown(value=text_dropdown_value[0], choices=text_entities_for_drop, allow_custom_value=True, interactive=True)
+
+    page_entities_for_drop = update_dropdown_list_based_on_dataframe(filtered_df, "page")
+    page_entities_drop = gr.Dropdown(value=page_dropdown_value[0], choices=page_entities_for_drop, allow_custom_value=True, interactive=True)
+
+    return recogniser_entities_drop, text_entities_drop, page_entities_drop
+
 def update_entities_df_recogniser_entities(choice:str, df:pd.DataFrame, page_dropdown_value:str, text_dropdown_value:str):
     '''
     Update the rows in a dataframe depending on the user choice from a dropdown
     '''
+
     if isinstance(choice, str):
         choice = [choice]
     if isinstance(page_dropdown_value, str):
@@ -592,20 +736,86 @@ def update_entities_df_text(choice:str, df:pd.DataFrame, label_dropdown_value:st
 
     return filtered_df, recogniser_entities_drop, page_entities_drop
     
-def reset_dropdowns():
+def reset_dropdowns(df:pd.DataFrame):
     '''
     Return Gradio dropdown objects with value 'ALL'.
     '''
-    return gr.Dropdown(value="ALL", allow_custom_value=True), gr.Dropdown(value="ALL", allow_custom_value=True), gr.Dropdown(value="ALL", allow_custom_value=True)
+
+    recogniser_entities_for_drop = update_dropdown_list_based_on_dataframe(df, "label")
+    recogniser_entities_drop = gr.Dropdown(value="ALL", choices=recogniser_entities_for_drop, allow_custom_value=True, interactive=True)    
+
+    text_entities_for_drop = update_dropdown_list_based_on_dataframe(df, "text")
+    text_entities_drop = gr.Dropdown(value="ALL", choices=text_entities_for_drop, allow_custom_value=True, interactive=True)
+
+    page_entities_for_drop = update_dropdown_list_based_on_dataframe(df, "page")
+    page_entities_drop = gr.Dropdown(value="ALL", choices=page_entities_for_drop, allow_custom_value=True, interactive=True)
+
+    return recogniser_entities_drop, text_entities_drop, page_entities_drop
     
 def df_select_callback(df: pd.DataFrame, evt: gr.SelectData):
 
         row_value_page = evt.row_value[0] # This is the page number value
+        row_value_label = evt.row_value[1] # This is the label number value
+        row_value_text = evt.row_value[2] # This is the text number value
+
+        row_value_df = pd.DataFrame(data={"page":[row_value_page], "label":[row_value_label], "text":[row_value_text]})
+
+        return row_value_page, row_value_df
 
-        if isinstance(row_value_page, list):
-            row_value_page = row_value_page[0]
+def df_select_callback_cost(df: pd.DataFrame, evt: gr.SelectData):
 
-        return row_value_page
+        row_value_code = evt.row_value[0] # This is the value for cost code
+        row_value_label = evt.row_value[1] # This is the label number value
+
+        #row_value_df = pd.DataFrame(data={"page":[row_value_code], "label":[row_value_label]})
+
+        return row_value_code
+
+def update_selected_review_df_row_colour(redaction_row_selection:pd.DataFrame, review_df:pd.DataFrame, colour:tuple=(0,0,255)):
+    '''
+    Update the colour of a single redaction box based on the values in a selection row
+    '''
+    colour_tuple =  str(tuple(colour))
+
+    if "color" not in review_df.columns: review_df["color"] = None
+
+    # Reset existing highlight colours
+    review_df.loc[review_df["color"]==colour_tuple, "color"] = review_df.loc[review_df["color"]==colour_tuple, "color"].apply(lambda _: '(0, 0, 0)')
+
+    review_df = review_df.merge(redaction_row_selection, on=["page", "label", "text"], indicator=True, how="left")
+    review_df.loc[review_df["_merge"]=="both", "color"] =  review_df.loc[review_df["_merge"] == "both", "color"].apply(lambda _: '(0, 0, 255)')
+
+    review_df.drop("_merge", axis=1, inplace=True)
+
+    review_df.to_csv(OUTPUT_FOLDER + "review_df_in_update_selected_review.csv")
+
+    return review_df
+
+def update_boxes_color(images: list, redaction_row_selection: pd.DataFrame, colour: tuple = (0, 255, 0)):
+    """
+    Update the color of bounding boxes in the images list based on redaction_row_selection.
+    
+    Parameters:
+    - images (list): List of dictionaries containing image paths and box metadata.
+    - redaction_row_selection (pd.DataFrame): DataFrame with 'page', 'label', and optionally 'text' columns.
+    - colour (tuple): RGB tuple for the new color.
+    
+    Returns:
+    - Updated list with modified colors.
+    """
+    # Convert DataFrame to a set for fast lookup
+    selection_set = set(zip(redaction_row_selection["page"], redaction_row_selection["label"]))
+
+    for page_idx, image_obj in enumerate(images):
+        if "boxes" in image_obj:
+            for box in image_obj["boxes"]:
+                if (page_idx, box["label"]) in selection_set:
+                    box["color"] = colour  # Update color
+    
+    return images
+
+def update_other_annotator_number_from_current(page_number_first_counter:int):
+    return page_number_first_counter
 
 def convert_image_coords_to_adobe(pdf_page_width:float, pdf_page_height:float, image_width:float, image_height:float, x1:float, y1:float, x2:float, y2:float):
     '''
@@ -644,27 +854,23 @@ def convert_image_coords_to_adobe(pdf_page_width:float, pdf_page_height:float, i
     
     return pdf_x1, pdf_y1, pdf_x2, pdf_y2
 
-def convert_pymupdf_coords_to_adobe(x1: float, y1: float, x2: float, y2: float):
+def convert_pymupdf_coords_to_adobe(x1: float, y1: float, x2: float, y2: float, pdf_page_height: float):
     """
     Converts coordinates from PyMuPDF (fitz) space to Adobe PDF space.
     
     Parameters:
-    - pdf_page_width: Width of the PDF page
-    - pdf_page_height: Height of the PDF page
     - x1, y1, x2, y2: Coordinates in PyMuPDF space
+    - pdf_page_height: Total height of the PDF page
     
     Returns:
     - Tuple of converted coordinates (x1, y1, x2, y2) in Adobe PDF space
     """
-    
-    # PyMuPDF and Adobe PDF coordinates are similar, but ensure y1 is always the lower value
-    pdf_x1, pdf_x2 = x1, x2
-    
-    # Ensure y1 is the bottom coordinate and y2 is the top
-    pdf_y1, pdf_y2 = min(y1, y2), max(y1, y2)
-    
-    return pdf_x1, pdf_y1, pdf_x2, pdf_y2
 
+    # PyMuPDF uses (0,0) at the bottom-left, while Adobe uses (0,0) at the top-left
+    adobe_y1 = pdf_page_height - y2  # Convert top coordinate
+    adobe_y2 = pdf_page_height - y1  # Convert bottom coordinate
+    
+    return x1, adobe_y1, x2, adobe_y2
 
 def create_xfdf(review_file_df:pd.DataFrame, pdf_path:str, pymupdf_doc:object, image_paths:List[str], document_cropboxes:List=[], page_sizes:List[dict]=[]):
     '''
@@ -687,45 +893,49 @@ def create_xfdf(review_file_df:pd.DataFrame, pdf_path:str, pymupdf_doc:object, i
         page_sizes_df = pd.DataFrame(page_sizes)
 
         # If there are no image coordinates, then convert coordinates to pymupdf coordinates prior to export
-        if len(page_sizes_df.loc[page_sizes_df["image_width"].isnull(),"image_width"]) == len(page_sizes_df["image_width"]):
-            print("No image dimensions found, using pymupdf coordinates for conversion.")
+        #if len(page_sizes_df.loc[page_sizes_df["image_width"].isnull(),"image_width"]) == len(page_sizes_df["image_width"]):
+        print("Using pymupdf coordinates for conversion.")
 
-            if "mediabox_width" not in review_file_df.columns:            
-                    review_file_df = review_file_df.merge(page_sizes_df, how="left", on = "page")
-            
-            # If all coordinates are less or equal to one, this is a relative page scaling - change back to image coordinates
-            if review_file_df["xmin"].max() <= 1 and review_file_df["xmax"].max() <= 1 and review_file_df["ymin"].max() <= 1 and review_file_df["ymax"].max() <= 1:
-                review_file_df["xmin"] = review_file_df["xmin"] * review_file_df["mediabox_width"]
-                review_file_df["xmax"] = review_file_df["xmax"] * review_file_df["mediabox_width"]
-                review_file_df["ymin"] = review_file_df["ymin"] * review_file_df["mediabox_height"]
-                review_file_df["ymax"] = review_file_df["ymax"] * review_file_df["mediabox_height"]
+        pages_are_images = False
 
-            pages_are_images = False
+        if "mediabox_width" not in review_file_df.columns:            
+                review_file_df = review_file_df.merge(page_sizes_df, how="left", on = "page")
+        
+        # If all coordinates are less or equal to one, this is a relative page scaling - change back to image coordinates
+        if review_file_df["xmin"].max() <= 1 and review_file_df["xmax"].max() <= 1 and review_file_df["ymin"].max() <= 1 and review_file_df["ymax"].max() <= 1:
+            review_file_df["xmin"] = review_file_df["xmin"] * review_file_df["mediabox_width"]
+            review_file_df["xmax"] = review_file_df["xmax"] * review_file_df["mediabox_width"]
+            review_file_df["ymin"] = review_file_df["ymin"] * review_file_df["mediabox_height"]
+            review_file_df["ymax"] = review_file_df["ymax"] * review_file_df["mediabox_height"]
+
+        # If all nulls, then can do image coordinate conversion
+        if len(page_sizes_df.loc[page_sizes_df["mediabox_width"].isnull(),"mediabox_width"]) == len(page_sizes_df["mediabox_width"]):
+
+            pages_are_images = True
 
-        # If no nulls, then can do image coordinate conversion
-        elif len(page_sizes_df.loc[page_sizes_df["image_width"].isnull(),"image_width"]) == 0:
+            review_file_df = multiply_coordinates_by_page_sizes(review_file_df, page_sizes_df, xmin="xmin", xmax="xmax", ymin="ymin", ymax="ymax")
 
-            if "image_width" not in review_file_df.columns:            
-                    review_file_df = review_file_df.merge(page_sizes_df, how="left", on = "page")
+            # if "image_width" not in review_file_df.columns:            
+            #         review_file_df = review_file_df.merge(page_sizes_df, how="left", on = "page")
             
-            # If all coordinates are less or equal to one, this is a relative page scaling - change back to image coordinates
-            if review_file_df["xmin"].max() <= 1 and review_file_df["xmax"].max() <= 1 and review_file_df["ymin"].max() <= 1 and review_file_df["ymax"].max() <= 1:
-                review_file_df["xmin"] = review_file_df["xmin"] * review_file_df["image_width"]
-                review_file_df["xmax"] = review_file_df["xmax"] * review_file_df["image_width"]
-                review_file_df["ymin"] = review_file_df["ymin"] * review_file_df["image_height"]
-                review_file_df["ymax"] = review_file_df["ymax"] * review_file_df["image_height"]
-
-                pages_are_images = True
+            # # If all coordinates are less or equal to one, this is a relative page scaling - change back to image coordinates
+            # if review_file_df["xmin"].max() <= 1 and review_file_df["xmax"].max() <= 1 and review_file_df["ymin"].max() <= 1 and review_file_df["ymax"].max() <= 1:
+            #     review_file_df["xmin"] = review_file_df["xmin"] * review_file_df["image_width"]
+            #     review_file_df["xmax"] = review_file_df["xmax"] * review_file_df["image_width"]
+            #     review_file_df["ymin"] = review_file_df["ymin"] * review_file_df["image_height"]
+            #     review_file_df["ymax"] = review_file_df["ymax"] * review_file_df["image_height"]
+
+                
     
     # Go through each row of the review_file_df, create an entry in the output Adobe xfdf file.
     for _, row in review_file_df.iterrows():
+        page_num_reported = row["page"]
         page_python_format = int(row["page"])-1
 
         pymupdf_page = pymupdf_doc.load_page(page_python_format)
 
         # Load cropbox sizes. Set cropbox to the original cropbox sizes from when the document was loaded into the app.
         if document_cropboxes:
-            #print("Document cropboxes:", document_cropboxes)
 
             # Extract numbers safely using regex
             match = re.findall(r"[-+]?\d*\.\d+|\d+", document_cropboxes[page_python_format])
@@ -740,14 +950,30 @@ def create_xfdf(review_file_df:pd.DataFrame, pdf_path:str, pymupdf_doc:object, i
 
         
         pdf_page_height = pymupdf_page.mediabox.height
-        pdf_page_width = pymupdf_page.mediabox.width 
+        pdf_page_width = pymupdf_page.mediabox.width
 
-        image = image_paths[page_python_format]
+        # Check if image dimensions for page exist in page_sizes_df
+        # image_dimensions = {}
 
-        if isinstance(image, str):
-            image = Image.open(image)
+        # image_dimensions['image_width'] = page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_width"].max()
+        # image_dimensions['image_height'] = page_sizes_df.loc[page_sizes_df['page']==page_num_reported, "image_height"].max()
 
-        image_page_width, image_page_height = image.size
+        # if pd.isna(image_dimensions['image_width']):
+        #     image_dimensions = {}
+
+        # image = image_paths[page_python_format]
+
+        # if image_dimensions:
+        #     image_page_width, image_page_height = image_dimensions["image_width"], image_dimensions["image_height"]
+        # if isinstance(image, str) and 'placeholder' not in image:
+        #     image = Image.open(image)
+        #     image_page_width, image_page_height = image.size
+        # else:
+        #     try:
+        #         image = Image.open(image)
+        #         image_page_width, image_page_height = image.size
+        #     except Exception as e:
+        #         print("Could not get image sizes due to:", e)        
 
         # Create redaction annotation
         redact_annot = SubElement(annots, 'redact')
@@ -759,23 +985,23 @@ def create_xfdf(review_file_df:pd.DataFrame, pdf_path:str, pymupdf_doc:object, i
         # Set page number (subtract 1 as PDF pages are 0-based)
         redact_annot.set('page', str(int(row['page']) - 1))
         
-        # Convert coordinates
-        if pages_are_images == True:
-            x1, y1, x2, y2 = convert_image_coords_to_adobe(
-                pdf_page_width,
-                pdf_page_height,
-                image_page_width,
-                image_page_height,
-                row['xmin'],
-                row['ymin'],
-                row['xmax'],
-                row['ymax']
-            )
-        else:
-            x1, y1, x2, y2 = convert_pymupdf_coords_to_adobe(row['xmin'],
-                row['ymin'],
-                row['xmax'],
-                row['ymax'])
+        # # Convert coordinates
+        # if pages_are_images == True:
+        #     x1, y1, x2, y2 = convert_image_coords_to_adobe(
+        #         pdf_page_width,
+        #         pdf_page_height,
+        #         image_page_width,
+        #         image_page_height,
+        #         row['xmin'],
+        #         row['ymin'],
+        #         row['xmax'],
+        #         row['ymax']
+        #     )
+        # else:
+        x1, y1, x2, y2 = convert_pymupdf_coords_to_adobe(row['xmin'],
+            row['ymin'],
+            row['xmax'],
+            row['ymax'], pdf_page_height)
 
         if CUSTOM_BOX_COLOUR == "grey":
             colour_str = "0.5,0.5,0.5"        
@@ -827,7 +1053,7 @@ def create_xfdf(review_file_df:pd.DataFrame, pdf_path:str, pymupdf_doc:object, i
     
     return xml_str
 
-def convert_df_to_xfdf(input_files:List[str], pdf_doc:Document, image_paths:List[str], output_folder:str = output_folder, document_cropboxes:List=[], page_sizes:List[dict]=[]):
+def convert_df_to_xfdf(input_files:List[str], pdf_doc:Document, image_paths:List[str], output_folder:str = OUTPUT_FOLDER, document_cropboxes:List=[], page_sizes:List[dict]=[]):
     '''
     Load in files to convert a review file into an Adobe comment file format
     '''
@@ -934,8 +1160,6 @@ def parse_xfdf(xfdf_path:str):
     # Find all redact elements using the namespace
     for redact in root.findall('.//xfdf:redact', namespaces=namespace):
 
-        #print("redact:", redact)
-
         redaction_info = {
             'image': '', # Image will be filled in later
             'page': int(redact.get('page')) + 1,  # Convert to 1-based index
@@ -948,12 +1172,10 @@ def parse_xfdf(xfdf_path:str):
             'color': redact.get('border-color', '(0, 0, 0)')  # Default to black if not specified
         }
         redactions.append(redaction_info)
-
-        print("redactions:", redactions)
     
     return redactions
 
-def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_paths:List[str], output_folder:str=output_folder):
+def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_paths:List[str], output_folder:str=OUTPUT_FOLDER):
     '''
     Convert redaction annotations from XFDF and associated images into a DataFrame.
     
@@ -969,8 +1191,6 @@ def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_path
     xfdf_paths = []
     df = pd.DataFrame()
 
-    #print("Image paths:", image_paths)
-
     # Sort the file paths so that the pdfs come first
     file_paths_list = sorted(file_paths_list, key=lambda x: (os.path.splitext(x)[1] != '.pdf', os.path.splitext(x)[1] != '.json'))
     
@@ -986,7 +1206,6 @@ def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_path
 
         if file_path_end == "pdf":
             pdf_name = os.path.basename(file_path)
-            #print("pymupdf_doc:", pymupdf_doc)
 
             # Add pdf to outputs
             output_paths.append(file_path)
@@ -997,18 +1216,10 @@ def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_path
                 message = "Original PDF needed to convert from .xfdf format"
                 print(message)
                 raise ValueError(message)
-
             xfdf_path = file
 
-            # if isinstance(xfdf_paths, str):
-            #     xfdf_path = xfdf_paths.name
-            # else:
-            #     xfdf_path = xfdf_paths[0].name
-
             file_path_name = get_file_name_without_type(xfdf_path)
 
-            #print("file_path_name:", file_path_name)
-
             # Parse the XFDF file
             redactions = parse_xfdf(xfdf_path)
             
@@ -1027,8 +1238,6 @@ def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_path
 
                 image_path = image_paths[page_python_format]
 
-                #print("image_path:", image_path)
-
                 if isinstance(image_path, str):
                     image = Image.open(image_path)
 
@@ -1040,7 +1249,6 @@ def convert_xfdf_to_dataframe(file_paths_list:List[str], pymupdf_doc, image_path
                 df.loc[_, ['xmin', 'ymin', 'xmax', 'ymax']] = [image_x1, image_y1, image_x2, image_y2]
             
                 # Optionally, you can add the image path or other relevant information
-                #print("Image path:", image_path)
                 df.loc[_, 'image'] = image_path
 
                 #print('row:', row)

tools/textract_batch_call.py

@@ -0,0 +1,308 @@
+import boto3
+import time
+import os
+import json
+import logging
+from urllib.parse import urlparse
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def analyze_pdf_with_textract(
+    local_pdf_path: str,
+    s3_bucket_name: str,
+    s3_input_prefix: str,
+    s3_output_prefix: str,
+    local_output_dir: str,
+    aws_region: str = None, # Optional: specify region if not default
+    poll_interval_seconds: int = 5,
+    max_polling_attempts: int = 120 # ~10 minutes total wait time
+    ):
+    """
+    Uploads a local PDF to S3, starts a Textract analysis job (detecting text & signatures),
+    waits for completion, and downloads the output JSON from S3 to a local directory.
+
+    Args:
+        local_pdf_path (str): Path to the local PDF file.
+        s3_bucket_name (str): Name of the S3 bucket to use.
+        s3_input_prefix (str): S3 prefix (folder) to upload the input PDF.
+        s3_output_prefix (str): S3 prefix (folder) where Textract should write output.
+        local_output_dir (str): Local directory to save the downloaded JSON results.
+        aws_region (str, optional): AWS region name. Defaults to boto3 default region.
+        poll_interval_seconds (int): Seconds to wait between polling Textract status.
+        max_polling_attempts (int): Maximum number of times to poll Textract status.
+
+    Returns:
+        str: Path to the downloaded local JSON output file, or None if failed.
+
+    Raises:
+        FileNotFoundError: If the local_pdf_path does not exist.
+        boto3.exceptions.NoCredentialsError: If AWS credentials are not found.
+        Exception: For other AWS errors or job failures.
+    """
+
+    if not os.path.exists(local_pdf_path):
+        raise FileNotFoundError(f"Input PDF not found: {local_pdf_path}")
+
+    if not os.path.exists(local_output_dir):
+        os.makedirs(local_output_dir)
+        logging.info(f"Created local output directory: {local_output_dir}")
+
+    # Initialize boto3 clients
+    session = boto3.Session(region_name=aws_region)
+    s3_client = session.client('s3')
+    textract_client = session.client('textract')
+
+    # --- 1. Upload PDF to S3 ---
+    pdf_filename = os.path.basename(local_pdf_path)
+    s3_input_key = os.path.join(s3_input_prefix, pdf_filename).replace("\\", "/") # Ensure forward slashes for S3
+
+    logging.info(f"Uploading '{local_pdf_path}' to 's3://{s3_bucket_name}/{s3_input_key}'...")
+    try:
+        s3_client.upload_file(local_pdf_path, s3_bucket_name, s3_input_key)
+        logging.info("Upload successful.")
+    except Exception as e:
+        logging.error(f"Failed to upload PDF to S3: {e}")
+        raise
+
+    # --- 2. Start Textract Document Analysis ---
+    logging.info("Starting Textract document analysis job...")
+    try:
+        response = textract_client.start_document_analysis(
+            DocumentLocation={
+                'S3Object': {
+                    'Bucket': s3_bucket_name,
+                    'Name': s3_input_key
+                }
+            },
+            FeatureTypes=['SIGNATURES', 'FORMS', 'TABLES'], # Analyze for signatures, forms, and tables
+            OutputConfig={
+                'S3Bucket': s3_bucket_name,
+                'S3Prefix': s3_output_prefix
+            }
+            # Optional: Add NotificationChannel for SNS topic notifications
+            # NotificationChannel={
+            #     'SNSTopicArn': 'YOUR_SNS_TOPIC_ARN',
+            #     'RoleArn': 'YOUR_IAM_ROLE_ARN_FOR_TEXTRACT_TO_ACCESS_SNS'
+            # }
+        )
+        job_id = response['JobId']
+        logging.info(f"Textract job started with JobId: {job_id}")
+
+    except Exception as e:
+        logging.error(f"Failed to start Textract job: {e}")
+        raise
+
+    # --- 3. Poll for Job Completion ---
+    job_status = 'IN_PROGRESS'
+    attempts = 0
+    logging.info("Polling Textract for job completion status...")
+
+    while job_status == 'IN_PROGRESS' and attempts < max_polling_attempts:
+        attempts += 1
+        try:
+            response = textract_client.get_document_analysis(JobId=job_id)
+            job_status = response['JobStatus']
+            logging.info(f"Polling attempt {attempts}/{max_polling_attempts}. Job status: {job_status}")
+
+            if job_status == 'IN_PROGRESS':
+                time.sleep(poll_interval_seconds)
+            elif job_status == 'SUCCEEDED':
+                logging.info("Textract job succeeded.")
+                break
+            elif job_status in ['FAILED', 'PARTIAL_SUCCESS']:
+                 status_message = response.get('StatusMessage', 'No status message provided.')
+                 warnings = response.get('Warnings', [])
+                 logging.error(f"Textract job ended with status: {job_status}. Message: {status_message}")
+                 if warnings:
+                     logging.warning(f"Warnings: {warnings}")
+                 # Decide if PARTIAL_SUCCESS should proceed or raise error
+                 # For simplicity here, we raise for both FAILED and PARTIAL_SUCCESS
+                 raise Exception(f"Textract job {job_id} failed or partially failed. Status: {job_status}. Message: {status_message}")
+            else:
+                # Should not happen based on documentation, but handle defensively
+                raise Exception(f"Unexpected Textract job status: {job_status}")
+
+        except textract_client.exceptions.InvalidJobIdException:
+             logging.error(f"Invalid JobId: {job_id}. This might happen if the job expired (older than 7 days) or never existed.")
+             raise
+        except Exception as e:
+             logging.error(f"Error while polling Textract status for job {job_id}: {e}")
+             raise
+
+    if job_status != 'SUCCEEDED':
+        raise TimeoutError(f"Textract job {job_id} did not complete successfully within the polling limit.")
+
+    # --- 4. Download Output JSON from S3 ---
+    # Textract typically creates output under s3_output_prefix/job_id/
+    # There might be multiple JSON files if pagination occurred during writing.
+    # Usually, for smaller docs, there's one file, often named '1'.
+    # For robust handling, list objects and find the JSON(s).
+
+    s3_output_key_prefix = os.path.join(s3_output_prefix, job_id).replace("\\", "/") + "/"
+    logging.info(f"Searching for output files in s3://{s3_bucket_name}/{s3_output_key_prefix}")
+
+    downloaded_file_path = None
+    try:
+        list_response = s3_client.list_objects_v2(
+            Bucket=s3_bucket_name,
+            Prefix=s3_output_key_prefix
+        )
+
+        output_files = list_response.get('Contents', [])
+        if not output_files:
+            # Sometimes Textract might take a moment longer to write the output after SUCCEEDED status
+            logging.warning("No output files found immediately after job success. Waiting briefly and retrying list...")
+            time.sleep(5)
+            list_response = s3_client.list_objects_v2(
+                Bucket=s3_bucket_name,
+                Prefix=s3_output_key_prefix
+            )
+            output_files = list_response.get('Contents', [])
+
+        if not output_files:
+             logging.error(f"No output files found in s3://{s3_bucket_name}/{s3_output_key_prefix}")
+             # You could alternatively try getting results via get_document_analysis pagination here
+             # but sticking to the request to download from S3 output path.
+             raise FileNotFoundError(f"Textract output files not found in S3 path: s3://{s3_bucket_name}/{s3_output_key_prefix}")
+
+        # Usually, we only need the first/main JSON output file(s)
+        # For simplicity, download the first one found. A more complex scenario might merge multiple files.
+        # Filter out potential directory markers if any key ends with '/'
+        json_files_to_download = [f for f in output_files if f['Key'] != s3_output_key_prefix and not f['Key'].endswith('/')]
+
+        if not json_files_to_download:
+            logging.error(f"No JSON files found (only prefix marker?) in s3://{s3_bucket_name}/{s3_output_key_prefix}")
+            raise FileNotFoundError(f"Textract output JSON files not found in S3 path: s3://{s3_bucket_name}/{s3_output_key_prefix}")
+
+        # Let's download the first JSON found. Often it's the only one or the main one.
+        s3_output_key = json_files_to_download[0]['Key']
+        output_filename_base = os.path.basename(pdf_filename).replace('.pdf', '')
+        local_output_filename = f"{output_filename_base}_textract_output_{job_id}.json"
+        local_output_path = os.path.join(local_output_dir, local_output_filename)
+
+        logging.info(f"Downloading Textract output from 's3://{s3_bucket_name}/{s3_output_key}' to '{local_output_path}'...")
+        s3_client.download_file(s3_bucket_name, s3_output_key, local_output_path)
+        logging.info("Download successful.")
+        downloaded_file_path = local_output_path
+
+        # Log if multiple files were found, as user might need to handle them
+        if len(json_files_to_download) > 1:
+            logging.warning(f"Multiple output files found in S3 output location. Downloaded the first: '{s3_output_key}'. Other files exist.")
+
+    except Exception as e:
+        logging.error(f"Failed to download or process Textract output from S3: {e}")
+        raise
+
+    return downloaded_file_path
+
+# --- Example Usage ---
+if __name__ == '__main__':
+    # --- Configuration --- (Replace with your actual values)
+    MY_LOCAL_PDF = r"C:\path\to\your\document.pdf" # Use raw string for Windows paths
+    MY_S3_BUCKET = "your-textract-demo-bucket-name" # MUST BE UNIQUE GLOBALLY
+    MY_S3_INPUT_PREFIX = "textract-inputs"          # Folder in the bucket for uploads
+    MY_S3_OUTPUT_PREFIX = "textract-outputs"        # Folder in the bucket for results
+    MY_LOCAL_OUTPUT_DIR = "./textract_results"      # Local folder to save JSON
+    MY_AWS_REGION = "us-east-1"                     # e.g., 'us-east-1', 'eu-west-1'
+
+    # --- Create a dummy PDF for testing if you don't have one ---
+    # Requires 'reportlab' library: pip install reportlab
+    try:
+        from reportlab.pdfgen import canvas
+        from reportlab.lib.pagesizes import letter
+        if not os.path.exists(MY_LOCAL_PDF):
+             print(f"Creating dummy PDF: {MY_LOCAL_PDF}")
+             c = canvas.Canvas(MY_LOCAL_PDF, pagesize=letter)
+             c.drawString(100, 750, "This is a test document for AWS Textract.")
+             c.drawString(100, 700, "It includes some text and a placeholder for a signature.")
+             c.drawString(100, 650, "Signed:")
+             # Draw a simple line/scribble for signature placeholder
+             c.line(150, 630, 250, 645)
+             c.line(250, 645, 300, 620)
+             c.save()
+             print("Dummy PDF created.")
+    except ImportError:
+        if not os.path.exists(MY_LOCAL_PDF):
+            print(f"Warning: reportlab not installed and '{MY_LOCAL_PDF}' not found. Cannot run example without an input PDF.")
+            exit() # Exit if no PDF available for the example
+    except Exception as e:
+         print(f"Error creating dummy PDF: {e}")
+         exit()
+
+
+    # --- Run the analysis ---
+    try:
+        output_json_path = analyze_pdf_with_textract(
+            local_pdf_path=MY_LOCAL_PDF,
+            s3_bucket_name=MY_S3_BUCKET,
+            s3_input_prefix=MY_S3_INPUT_PREFIX,
+            s3_output_prefix=MY_S3_OUTPUT_PREFIX,
+            local_output_dir=MY_LOCAL_OUTPUT_DIR,
+            aws_region=MY_AWS_REGION
+        )
+
+        if output_json_path:
+            print(f"\n--- Analysis Complete ---")
+            print(f"Textract output JSON saved to: {output_json_path}")
+
+            # Optional: Load and print some info from the JSON
+            with open(output_json_path, 'r') as f:
+                results = json.load(f)
+            print(f"Detected {results.get('DocumentMetadata', {}).get('Pages', 'N/A')} page(s).")
+            # Find signature blocks (Note: This is basic, real parsing might be more complex)
+            signature_blocks = [block for block in results.get('Blocks', []) if block.get('BlockType') == 'SIGNATURE']
+            print(f"Found {len(signature_blocks)} potential signature block(s).")
+            if signature_blocks:
+                 print(f"First signature confidence: {signature_blocks[0].get('Confidence', 'N/A')}")
+
+
+    except FileNotFoundError as e:
+        print(f"\nError: Input file not found. {e}")
+    except Exception as e:
+        print(f"\nAn error occurred during the process: {e}")
+
+import boto3
+import time
+import os
+
+def download_textract_output(job_id, output_bucket, output_prefix, local_folder):
+    """
+    Checks the status of a Textract job and downloads the output ZIP file if the job is complete.
+
+    :param job_id: The Textract job ID.
+    :param output_bucket: The S3 bucket where the output is stored.
+    :param output_prefix: The prefix (folder path) in S3 where the output file is stored.
+    :param local_folder: The local directory where the ZIP file should be saved.
+    """
+    textract_client = boto3.client('textract')
+    s3_client = boto3.client('s3')
+
+    # Check job status
+    while True:
+        response = textract_client.get_document_analysis(JobId=job_id)
+        status = response['JobStatus']
+        
+        if status == 'SUCCEEDED':
+            print("Job completed successfully.")
+            break
+        elif status == 'FAILED':
+            print("Job failed:", response.get("StatusMessage", "No error message provided."))
+            return
+        else:
+            print(f"Job is still {status}, waiting...")
+            time.sleep(10)  # Wait before checking again
+
+    # Find output ZIP file in S3
+    output_file_key = f"{output_prefix}/{job_id}.zip"
+    local_file_path = os.path.join(local_folder, f"{job_id}.zip")
+
+    # Download file
+    try:
+        s3_client.download_file(output_bucket, output_file_key, local_file_path)
+        print(f"Output file downloaded to: {local_file_path}")
+    except Exception as e:
+        print(f"Error downloading file: {e}")
+
+# Example usage:
+# download_textract_output("your-job-id", "your-output-bucket", "your-output-prefix", "/path/to/local/folder")