Added examples to tops of various tabs to demonstrate basic functions (optional). Minor changes to example csv ocr output

app.py

@@ -1,5 +1,4 @@
 import os
-import time
 
 import gradio as gr
 import pandas as pd
@@ -95,6 +94,7 @@ from tools.config import (
     SAVE_LOGS_TO_DYNAMODB,
     SESSION_OUTPUT_FOLDER,
     SHOW_COSTS,
+    SHOW_EXAMPLES,
     SHOW_LANGUAGE_SELECTION,
     SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS,
     TABULAR_PII_DETECTION_MODELS,
@@ -206,8 +206,11 @@ pd.set_option("future.no_silent_downcasting", True)
 ensure_folder_exists(CONFIG_FOLDER)
 ensure_folder_exists(OUTPUT_FOLDER)
 ensure_folder_exists(INPUT_FOLDER)
-ensure_folder_exists(GRADIO_TEMP_DIR)
-ensure_folder_exists(MPLCONFIGDIR)
+if GRADIO_TEMP_DIR:
+    ensure_folder_exists(GRADIO_TEMP_DIR)
+if MPLCONFIGDIR:
+    ensure_folder_exists(MPLCONFIGDIR)
+
 ensure_folder_exists(FEEDBACK_LOGS_FOLDER)
 ensure_folder_exists(ACCESS_LOGS_FOLDER)
 ensure_folder_exists(USAGE_LOGS_FOLDER)
@@ -291,79 +294,116 @@ if DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX:
 CHOSEN_COMPREHEND_ENTITIES.extend(custom_entities)
 FULL_COMPREHEND_ENTITY_LIST.extend(custom_entities)
 
-FILE_INPUT_HEIGHT = int(FILE_INPUT_HEIGHT)
-
-
-# Wrapper functions to add timing to deduplication functions
-def run_duplicate_analysis_with_timing(
-    files,
-    threshold,
-    min_words,
-    min_consecutive,
-    greedy_match,
-    combine_pages,
-    output_folder,
-):
-    """
-    Wrapper for run_duplicate_analysis that adds timing and returns time taken.
-    """
-    start_time = time.time()
-    results_df, output_paths, full_data_by_file = run_duplicate_analysis(
-        files=files,
-        threshold=threshold,
-        min_words=min_words,
-        min_consecutive=min_consecutive,
-        greedy_match=greedy_match,
-        combine_pages=combine_pages,
-        output_folder=output_folder,
-    )
-    end_time = time.time()
-    processing_time = end_time - start_time
-
-    # Store the time taken in a global variable for logging
-    global duplicate_analysis_time_taken
-    duplicate_analysis_time_taken = processing_time
-
-    return results_df, output_paths, full_data_by_file
-
-
-def run_tabular_duplicate_detection_with_timing(
-    files,
-    threshold,
-    min_words,
-    text_columns,
-    output_folder,
-    do_initial_clean_dup,
-    in_excel_tabular_sheets,
-    remove_duplicate_rows,
-):
-    """
-    Wrapper for run_tabular_duplicate_detection that adds timing and returns time taken.
-    """
-    start_time = time.time()
-    results_df, output_paths, file_choices = run_tabular_duplicate_detection(
-        files=files,
-        threshold=threshold,
-        min_words=min_words,
-        text_columns=text_columns,
-        output_folder=output_folder,
-        do_initial_clean_dup=do_initial_clean_dup,
-        in_excel_tabular_sheets=in_excel_tabular_sheets,
-        remove_duplicate_rows=remove_duplicate_rows,
-    )
-    end_time = time.time()
-    processing_time = end_time - start_time
-
-    # Store the time taken in a global variable for logging
-    global tabular_duplicate_analysis_time_taken
-    tabular_duplicate_analysis_time_taken = processing_time
-
-    return results_df, output_paths, file_choices
-
-
-# Initialize global variables for timing
-duplicate_analysis_time_taken = 0.0
-tabular_duplicate_analysis_time_taken = 0.0
+# Load some components outside of blocks context that are used for examples
+## Redaction examples
+in_doc_files = gr.File(
+    label="Choose a PDF document or image file (PDF, JPG, PNG)",
+    file_count="multiple",
+    file_types=[".pdf", ".jpg", ".png", ".json", ".zip"],
+    height=FILE_INPUT_HEIGHT,
+)
+
+text_extract_method_radio = gr.Radio(
+    label="""Choose text extraction method. Local options are lower quality but cost nothing - they may be worth a try if you are willing to spend some time reviewing outputs. AWS Textract has a cost per page - £2.66 ($3.50) per 1,000 pages with signature detection (default), £1.14 ($1.50) without. Change the settings in the tab below (AWS Textract signature detection) to change this.""",
+    value=DEFAULT_TEXT_EXTRACTION_MODEL,
+    choices=TEXT_EXTRACTION_MODELS,
+)
+
+pii_identification_method_drop = gr.Radio(
+    label="""Choose personal information detection method. The local model is lower quality but costs nothing - it may be worth a try if you are willing to spend some time reviewing outputs, or if you are only interested in searching for custom search terms (see Redaction settings - custom deny list). AWS Comprehend has a cost of around £0.0075 ($0.01) per 10,000 characters.""",
+    value=DEFAULT_PII_DETECTION_MODEL,
+    choices=PII_DETECTION_MODELS,
+)
+
+handwrite_signature_checkbox = gr.CheckboxGroup(
+    label="AWS Textract extraction settings",
+    choices=HANDWRITE_SIGNATURE_TEXTBOX_FULL_OPTIONS,
+    value=DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX,
+)
+
+in_redact_entities = gr.Dropdown(
+    value=CHOSEN_REDACT_ENTITIES,
+    choices=FULL_ENTITY_LIST,
+    multiselect=True,
+    label="Local PII identification model (click empty space in box for full list)",
+)
+in_redact_comprehend_entities = gr.Dropdown(
+    value=CHOSEN_COMPREHEND_ENTITIES,
+    choices=FULL_COMPREHEND_ENTITY_LIST,
+    multiselect=True,
+    label="AWS Comprehend PII identification model (click empty space in box for full list)",
+)
+
+## Deduplication examples
+in_duplicate_pages = gr.File(
+    label="Upload one or multiple 'ocr_output.csv' files to find duplicate pages and subdocuments",
+    file_count="multiple",
+    height=FILE_INPUT_HEIGHT,
+    file_types=[".csv"],
+)
+
+duplicate_threshold_input = gr.Number(
+    value=DEFAULT_DUPLICATE_DETECTION_THRESHOLD,
+    label="Similarity threshold",
+    info="Score (0-1) to consider pages a match.",
+)
+
+min_word_count_input = gr.Number(
+    value=DEFAULT_MIN_WORD_COUNT,
+    label="Minimum word count",
+    info="Pages with fewer words than this value are ignored.",
+)
+
+combine_page_text_for_duplicates_bool = gr.Checkbox(
+    value=True,
+    label="Analyse duplicate text by page (off for by line)",
+)
+
+## Tabular examples
+in_data_files = gr.File(
+    label="Choose Excel or csv files",
+    file_count="multiple",
+    file_types=[".xlsx", ".xls", ".csv", ".parquet", ".docx"],
+    height=FILE_INPUT_HEIGHT,
+)
+
+in_colnames = gr.Dropdown(
+    choices=["Choose columns to anonymise"],
+    multiselect=True,
+    allow_custom_value=True,
+    label="Select columns that you want to anonymise (showing columns present across all files).",
+)
+
+pii_identification_method_drop_tabular = gr.Radio(
+    label="Choose PII detection method. AWS Comprehend has a cost of approximately $0.01 per 10,000 characters.",
+    value=DEFAULT_PII_DETECTION_MODEL,
+    choices=TABULAR_PII_DETECTION_MODELS,
+)
+
+anon_strategy = gr.Radio(
+    choices=[
+        "replace with 'REDACTED'",
+        "replace with <ENTITY_NAME>",
+        "redact completely",
+        "hash",
+        "mask",
+    ],
+    label="Select an anonymisation method.",
+    value=DEFAULT_TABULAR_ANONYMISATION_STRATEGY,
+)  # , "encrypt", "fake_first_name" are also available, but are not currently included as not that useful in current form
+
+in_tabular_duplicate_files = gr.File(
+    label="Upload CSV, Excel, or Parquet files to find duplicate cells/rows. Note that the app will remove duplicates from later cells/files that are found in earlier cells/files and not vice versa.",
+    file_count="multiple",
+    file_types=[".csv", ".xlsx", ".xls", ".parquet"],
+    height=FILE_INPUT_HEIGHT,
+)
+
+tabular_text_columns = gr.Dropdown(
+    label="Choose columns to deduplicate",
+    multiselect=True,
+    allow_custom_value=True,
+)
 
 # Create the gradio interface
 app = gr.Blocks(
@@ -967,35 +1007,105 @@ with app:
     # REDACTION PDF/IMAGES TABLE
     ###
     with gr.Tab("Redact PDFs/images"):
-        with gr.Accordion("Redact document", open=True):
-            in_doc_files = gr.File(
-                label="Choose a PDF document or image file (PDF, JPG, PNG)",
-                file_count="multiple",
-                file_types=[".pdf", ".jpg", ".png", ".json", ".zip"],
-                height=FILE_INPUT_HEIGHT,
-            )
 
-            text_extract_method_radio = gr.Radio(
-                label="""Choose text extraction method. Local options are lower quality but cost nothing - they may be worth a try if you are willing to spend some time reviewing outputs. AWS Textract has a cost per page - £2.66 ($3.50) per 1,000 pages with signature detection (default), £1.14 ($1.50) without. Change the settings in the tab below (AWS Textract signature detection) to change this.""",
-                value=DEFAULT_TEXT_EXTRACTION_MODEL,
-                choices=TEXT_EXTRACTION_MODELS,
+        # Examples for PDF/image redaction
+        if SHOW_EXAMPLES == "True":
+            gr.Markdown(
+                "### Try an example - Click on an example below and then the 'Extract text and redact document' button:"
+            )
+            redaction_examples = gr.Examples(
+                examples=[
+                    [
+                        [
+                            "example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+                        ],
+                        "Local model - selectable text",
+                        "Local",
+                        [],
+                        CHOSEN_REDACT_ENTITIES,
+                        CHOSEN_COMPREHEND_ENTITIES,
+                        [
+                            "example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+                        ],
+                    ],
+                    [
+                        ["example_data/example_complaint_letter.jpg"],
+                        "Local OCR model - PDFs without selectable text",
+                        "Local",
+                        [],
+                        CHOSEN_REDACT_ENTITIES,
+                        CHOSEN_COMPREHEND_ENTITIES,
+                        ["example_data/example_complaint_letter.jpg"],
+                    ],
+                    [
+                        ["example_data/graduate-job-example-cover-letter.pdf"],
+                        "Local OCR model - PDFs without selectable text",
+                        "Local",
+                        [],
+                        ["TITLES", "PERSON", "DATE_TIME"],
+                        CHOSEN_COMPREHEND_ENTITIES,
+                        ["example_data/graduate-job-example-cover-letter.pdf"],
+                    ],
+                    [
+                        ["example_data/Partnership-Agreement-Toolkit_0_0.pdf"],
+                        "AWS Textract service - all PDF types",
+                        "AWS Comprehend",
+                        ["Extract handwriting", "Extract signatures"],
+                        CHOSEN_REDACT_ENTITIES,
+                        CHOSEN_COMPREHEND_ENTITIES,
+                        ["example_data/Partnership-Agreement-Toolkit_0_0.pdf"],
+                    ],
+                ],
+                inputs=[
+                    in_doc_files,
+                    text_extract_method_radio,
+                    pii_identification_method_drop,
+                    handwrite_signature_checkbox,
+                    in_redact_entities,
+                    in_redact_comprehend_entities,
+                    prepared_pdf_state,
+                ],
+                example_labels=[
+                    "PDF with selectable text redaction",
+                    "Image redaction with local OCR",
+                    "PDF redaction with custom entities (TITLES, PERSON, DATE_TIME)",
+                    "PDF redaction with AWS services and signature detection",
+                ],
             )
 
+        with gr.Accordion("Redact document", open=True):
+            # in_doc_files = gr.File(
+            #     label="Choose a PDF document or image file (PDF, JPG, PNG)",
+            #     file_count="multiple",
+            #     file_types=[".pdf", ".jpg", ".png", ".json", ".zip"],
+            #     height=FILE_INPUT_HEIGHT,
+            # )
+            in_doc_files.render()
+
+            # text_extract_method_radio = gr.Radio(
+            #     label="""Choose text extraction method. Local options are lower quality but cost nothing - they may be worth a try if you are willing to spend some time reviewing outputs. AWS Textract has a cost per page - £2.66 ($3.50) per 1,000 pages with signature detection (default), £1.14 ($1.50) without. Change the settings in the tab below (AWS Textract signature detection) to change this.""",
+            #     value=DEFAULT_TEXT_EXTRACTION_MODEL,
+            #     choices=TEXT_EXTRACTION_MODELS,
+            # )
+            text_extract_method_radio.render()
+
             with gr.Accordion(
                 "Enable AWS Textract signature detection (default is off)", open=False
             ):
-                handwrite_signature_checkbox = gr.CheckboxGroup(
-                    label="AWS Textract extraction settings",
-                    choices=HANDWRITE_SIGNATURE_TEXTBOX_FULL_OPTIONS,
-                    value=DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX,
-                )
+                # handwrite_signature_checkbox = gr.CheckboxGroup(
+                #     label="AWS Textract extraction settings",
+                #     choices=HANDWRITE_SIGNATURE_TEXTBOX_FULL_OPTIONS,
+                #     value=DEFAULT_HANDWRITE_SIGNATURE_CHECKBOX,
+                # )
+                handwrite_signature_checkbox.render()
 
             with gr.Row(equal_height=True):
-                pii_identification_method_drop = gr.Radio(
-                    label="""Choose personal information detection method. The local model is lower quality but costs nothing - it may be worth a try if you are willing to spend some time reviewing outputs, or if you are only interested in searching for custom search terms (see Redaction settings - custom deny list). AWS Comprehend has a cost of around £0.0075 ($0.01) per 10,000 characters.""",
-                    value=DEFAULT_PII_DETECTION_MODEL,
-                    choices=PII_DETECTION_MODELS,
-                )
+                # pii_identification_method_drop = gr.Radio(
+                #     label="""Choose personal information detection method. The local model is lower quality but costs nothing - it may be worth a try if you are willing to spend some time reviewing outputs, or if you are only interested in searching for custom search terms (see Redaction settings - custom deny list). AWS Comprehend has a cost of around £0.0075 ($0.01) per 10,000 characters.""",
+                #     value=DEFAULT_PII_DETECTION_MODEL,
+                #     choices=PII_DETECTION_MODELS,
+                # )
+                pii_identification_method_drop.render()
 
             if SHOW_COSTS == "True":
                 with gr.Accordion(
@@ -1536,30 +1646,72 @@ with app:
             "Search for duplicate pages/subdocuments in your ocr_output files. By default, this function will search for duplicate text across multiple pages, and then join consecutive matching pages together into matched 'subdocuments'. The results can be reviewed below, false positives removed, and then the verified results applied to a document you have loaded in on the 'Review redactions' tab."
         )
 
-        with gr.Accordion("Step 1: Configure and run analysis", open=True):
-            in_duplicate_pages = gr.File(
-                label="Upload one or multiple 'ocr_output.csv' files to find duplicate pages and subdocuments",
-                file_count="multiple",
-                height=FILE_INPUT_HEIGHT,
-                file_types=[".csv"],
+        # Examples for duplicate page detection
+        if SHOW_EXAMPLES == "True":
+            gr.Markdown(
+                "### Try an example - Click on an example below and then the 'Identify duplicate pages/subdocuments' button:"
+            )
+            duplicate_examples = gr.Examples(
+                examples=[
+                    [
+                        [
+                            "example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv"
+                        ],
+                        0.95,
+                        10,
+                        True,
+                    ],
+                    [
+                        [
+                            "example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv"
+                        ],
+                        0.95,
+                        3,
+                        False,
+                    ],
+                ],
+                inputs=[
+                    in_duplicate_pages,
+                    duplicate_threshold_input,
+                    min_word_count_input,
+                    combine_page_text_for_duplicates_bool,
+                ],
+                example_labels=[
+                    "Find duplicate pages of text in document OCR outputs",
+                    "Find duplicate text lines in document OCR outputs",
+                ],
             )
 
+        with gr.Accordion("Step 1: Configure and run analysis", open=True):
+            # in_duplicate_pages = gr.File(
+            #     label="Upload one or multiple 'ocr_output.csv' files to find duplicate pages and subdocuments",
+            #     file_count="multiple",
+            #     height=FILE_INPUT_HEIGHT,
+            #     file_types=[".csv"],
+            # )
+            in_duplicate_pages.render()
+
             with gr.Accordion("Duplicate matching parameters", open=False):
                 with gr.Row():
-                    duplicate_threshold_input = gr.Number(
-                        value=DEFAULT_DUPLICATE_DETECTION_THRESHOLD,
-                        label="Similarity threshold",
-                        info="Score (0-1) to consider pages a match.",
-                    )
-                    min_word_count_input = gr.Number(
-                        value=DEFAULT_MIN_WORD_COUNT,
-                        label="Minimum word count",
-                        info="Pages with fewer words than this value are ignored.",
-                    )
-                    combine_page_text_for_duplicates_bool = gr.Checkbox(
-                        value=True,
-                        label="Analyse duplicate text by page (off for by line)",
-                    )
+                    # duplicate_threshold_input = gr.Number(
+                    #     value=DEFAULT_DUPLICATE_DETECTION_THRESHOLD,
+                    #     label="Similarity threshold",
+                    #     info="Score (0-1) to consider pages a match.",
+                    # )
+                    duplicate_threshold_input.render()
+
+                    # min_word_count_input = gr.Number(
+                    #     value=DEFAULT_MIN_WORD_COUNT,
+                    #     label="Minimum word count",
+                    #     info="Pages with fewer words than this value are ignored.",
+                    # )
+                    min_word_count_input.render()
+
+                    # combine_page_text_for_duplicates_bool = gr.Checkbox(
+                    #     value=True,
+                    #     label="Analyse duplicate text by page (off for by line)",
+                    # )
+                    combine_page_text_for_duplicates_bool.render()
 
                 gr.Markdown("#### Matching Strategy")
                 greedy_match_input = gr.Checkbox(
@@ -1653,14 +1805,62 @@ with app:
             """Choose Word or a tabular data file (xlsx or csv) to redact. Note that when redacting complex Word files with e.g. images, some content/formatting will be removed, and it may not attempt to redact headers. You may prefer to convert the doc file to PDF in Word, and then run it through the first tab of this app (Print to PDF in print settings). Alternatively, an xlsx file output is provided when redacting docx files directly to allow for copying and pasting outputs back into the original document if preferred."""
         )
 
+        # Examples for Word/Excel/csv redaction and tabular duplicate detection
+        if SHOW_EXAMPLES == "True":
+            gr.Markdown(
+                "### Try an example - Click on an example below and then the 'Redact text/data files' button for redaction, or the 'Find duplicate cells/rows' button for duplicate detection:"
+            )
+            tabular_examples = gr.Examples(
+                examples=[
+                    [
+                        ["example_data/combined_case_notes.csv"],
+                        ["Case Note", "Client"],
+                        "Local",
+                        "replace with 'REDACTED'",
+                        ["example_data/combined_case_notes.csv"],
+                        ["Case Note"],
+                    ],
+                    [
+                        ["example_data/Bold minimalist professional cover letter.docx"],
+                        [],
+                        "Local",
+                        "replace with 'REDACTED'",
+                        [],
+                        [],
+                    ],
+                    [
+                        ["example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv"],
+                        ["text"],
+                        "Local",
+                        "replace with 'REDACTED'",
+                        ["example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv"],
+                        ["text"],
+                    ],
+                ],
+                inputs=[
+                    in_data_files,
+                    in_colnames,
+                    pii_identification_method_drop_tabular,
+                    anon_strategy,
+                    in_tabular_duplicate_files,
+                    tabular_text_columns,
+                ],
+                example_labels=[
+                    "CSV file redaction with specific columns - remove text",
+                    "Word document redaction - replace with REDACTED",
+                    "Tabular duplicate detection in CSV files",
+                ],
+            )
+
         with gr.Accordion("Redact Word or Excel/csv files", open=True):
             with gr.Accordion("Upload docx, xlsx, or csv files", open=True):
-                in_data_files = gr.File(
-                    label="Choose Excel or csv files",
-                    file_count="multiple",
-                    file_types=[".xlsx", ".xls", ".csv", ".parquet", ".docx"],
-                    height=FILE_INPUT_HEIGHT,
-                )
+                # in_data_files = gr.File(
+                #     label="Choose Excel or csv files",
+                #     file_count="multiple",
+                #     file_types=[".xlsx", ".xls", ".csv", ".parquet", ".docx"],
+                #     height=FILE_INPUT_HEIGHT,
+                # )
+                in_data_files.render()
             with gr.Accordion("Redact open text", open=False):
                 in_text = gr.Textbox(
                     label="Enter open text",
@@ -1676,34 +1876,39 @@ with app:
                 allow_custom_value=True,
             )
 
-            in_colnames = gr.Dropdown(
-                choices=["Choose columns to anonymise"],
-                multiselect=True,
-                label="Select columns that you want to anonymise (showing columns present across all files).",
-            )
-
-            pii_identification_method_drop_tabular = gr.Radio(
-                label="Choose PII detection method. AWS Comprehend has a cost of approximately $0.01 per 10,000 characters.",
-                value=DEFAULT_PII_DETECTION_MODEL,
-                choices=TABULAR_PII_DETECTION_MODELS,
-            )
+            # in_colnames = gr.Dropdown(
+            #     choices=["Choose columns to anonymise"],
+            #     multiselect=True,
+            #     allow_custom_value=True,
+            #     label="Select columns that you want to anonymise (showing columns present across all files).",
+            # )
+            in_colnames.render()
+
+            # pii_identification_method_drop_tabular = gr.Radio(
+            #     label="Choose PII detection method. AWS Comprehend has a cost of approximately $0.01 per 10,000 characters.",
+            #     value=DEFAULT_PII_DETECTION_MODEL,
+            #     choices=TABULAR_PII_DETECTION_MODELS,
+            # )
+            pii_identification_method_drop_tabular.render()
 
             with gr.Accordion(
                 "Anonymisation output format - by default will replace PII with a blank space",
                 open=False,
             ):
                 with gr.Row():
-                    anon_strategy = gr.Radio(
-                        choices=[
-                            "replace with 'REDACTED'",
-                            "replace with <ENTITY_NAME>",
-                            "redact completely",
-                            "hash",
-                            "mask",
-                        ],
-                        label="Select an anonymisation method.",
-                        value=DEFAULT_TABULAR_ANONYMISATION_STRATEGY,
-                    )  # , "encrypt", "fake_first_name" are also available, but are not currently included as not that useful in current form
+                    # anon_strategy = gr.Radio(
+                    #     choices=[
+                    #         "replace with 'REDACTED'",
+                    #         "replace with <ENTITY_NAME>",
+                    #         "redact completely",
+                    #         "hash",
+                    #         "mask",
+                    #     ],
+                    #     label="Select an anonymisation method.",
+                    #     value=DEFAULT_TABULAR_ANONYMISATION_STRATEGY,
+                    # )  # , "encrypt", "fake_first_name" are also available, but are not currently included as not that useful in current form
+                    anon_strategy.render()
+
                     do_initial_clean = gr.Checkbox(
                         label="Do initial clean of text (remove URLs, HTML tags, and non-ASCII characters)",
                         value=DO_INITIAL_TABULAR_DATA_CLEAN,
@@ -1713,15 +1918,15 @@ with app:
                 "Redact text/data files", variant="primary"
             )
 
-            with gr.Row():
-                text_output_summary = gr.Textbox(label="Output result", lines=4)
-                text_output_file = gr.File(label="Output files")
-                text_tabular_files_done = gr.Number(
-                    value=0,
-                    label="Number of tabular files redacted",
-                    interactive=False,
-                    visible=False,
-                )
+        with gr.Row():
+            text_output_summary = gr.Textbox(label="Output result", lines=4)
+            text_output_file = gr.File(label="Output files")
+            text_tabular_files_done = gr.Number(
+                value=0,
+                label="Number of tabular files redacted",
+                interactive=False,
+                visible=False,
+            )
 
         ###
         # TABULAR DUPLICATE DETECTION
@@ -1732,12 +1937,13 @@ with app:
             )
 
             with gr.Accordion("Step 1: Upload files and configure analysis", open=True):
-                in_tabular_duplicate_files = gr.File(
-                    label="Upload CSV, Excel, or Parquet files to find duplicate cells/rows. Note that the app will remove duplicates from later cells/files that are found in earlier cells/files and not vice versa.",
-                    file_count="multiple",
-                    file_types=[".csv", ".xlsx", ".xls", ".parquet"],
-                    height=FILE_INPUT_HEIGHT,
-                )
+                # in_tabular_duplicate_files = gr.File(
+                #     label="Upload CSV, Excel, or Parquet files to find duplicate cells/rows. Note that the app will remove duplicates from later cells/files that are found in earlier cells/files and not vice versa.",
+                #     file_count="multiple",
+                #     file_types=[".csv", ".xlsx", ".xls", ".parquet"],
+                #     height=FILE_INPUT_HEIGHT,
+                # )
+                in_tabular_duplicate_files.render()
 
                 with gr.Row(equal_height=True):
                     tabular_duplicate_threshold = gr.Number(
@@ -1768,12 +1974,13 @@ with app:
                         allow_custom_value=True,
                     )
 
-                    tabular_text_columns = gr.Dropdown(
-                        choices=DEFAULT_TEXT_COLUMNS,
-                        multiselect=True,
-                        label="Select specific columns to analyse (leave empty to analyse all text columns simultaneously - i.e. all text is joined together)",
-                        info="If no columns selected, all text columns will combined together and analysed",
-                    )
+                    # tabular_text_columns = gr.Dropdown(
+                    #     choices=DEFAULT_TEXT_COLUMNS,
+                    #     multiselect=True,
+                    #     label="Select specific columns to analyse (leave empty to analyse all text columns simultaneously - i.e. all text is joined together)",
+                    #     info="If no columns selected, all text columns will combined together and analysed",
+                    # )
+                    tabular_text_columns.render()
 
                 find_tabular_duplicates_btn = gr.Button(
                     value="Find duplicate cells/rows", variant="primary"
@@ -1937,18 +2144,20 @@ with app:
                         )
 
         with gr.Accordion("Select entity types to redact", open=True):
-            in_redact_entities = gr.Dropdown(
-                value=CHOSEN_REDACT_ENTITIES,
-                choices=FULL_ENTITY_LIST,
-                multiselect=True,
-                label="Local PII identification model (click empty space in box for full list)",
-            )
-            in_redact_comprehend_entities = gr.Dropdown(
-                value=CHOSEN_COMPREHEND_ENTITIES,
-                choices=FULL_COMPREHEND_ENTITY_LIST,
-                multiselect=True,
-                label="AWS Comprehend PII identification model (click empty space in box for full list)",
-            )
+            # in_redact_entities = gr.Dropdown(
+            #     value=CHOSEN_REDACT_ENTITIES,
+            #     choices=FULL_ENTITY_LIST,
+            #     multiselect=True,
+            #     label="Local PII identification model (click empty space in box for full list)",
+            # )
+            # in_redact_comprehend_entities = gr.Dropdown(
+            #     value=CHOSEN_COMPREHEND_ENTITIES,
+            #     choices=FULL_COMPREHEND_ENTITY_LIST,
+            #     multiselect=True,
+            #     label="AWS Comprehend PII identification model (click empty space in box for full list)",
+            # )
+            in_redact_entities.render()
+            in_redact_comprehend_entities.render()
 
             with gr.Row():
                 max_fuzzy_spelling_mistakes_num = gr.Number(
@@ -5013,6 +5222,7 @@ with app:
             comprehend_query_number,
         ],
         api_name="redact_data",
+        show_progress_on=[text_output_summary],
     )
 
     # If the output file count text box changes, keep going with redacting each data file until done
@@ -5053,6 +5263,7 @@ with app:
             actual_time_taken_number,
             comprehend_query_number,
         ],
+        show_progress_on=[text_output_summary],
     ).success(
         fn=reveal_feedback_buttons,
         outputs=[
@@ -5085,6 +5296,7 @@ with app:
             actual_time_taken_number,
             task_textbox,
         ],
+        show_progress_on=[results_df_preview],
     )
 
     # full_duplicated_data_df,

example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_ocr_output.csv

@@ -1,277 +1,277 @@
-page,text,left,top,width,height
-1,Partnership Agreement,0.516078,0.027879,0.440784,0.032424
-1,SisterCities,0.169804,0.033333,0.238431,0.028182
-1,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788
-1,Toolkit,0.830588,0.07303,0.126667,0.025152
-1,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303
-1,Types of Affiliations,0.117255,0.157576,0.241961,0.02
-1,Sister City Relationship,0.117647,0.187273,0.196863,0.013939
-1,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636
-1,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939
-1,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636
-1,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939
-1,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636
-1,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636
-1,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636
-1,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636
-1,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636
-1,Friendship City,0.118039,0.372121,0.127059,0.013939
-1,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636
-1,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242
-1,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636
-1,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636
-1,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636
-1,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333
-1,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636
-1,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636
-1,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636
-1,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333
-1,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727
-1,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636
-1,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333
-1,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939
-1,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242
-1,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636
-1,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636
-1,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939
-1,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939
-1,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636
-1,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939
-1,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242
-1,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333
-1,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939
-1,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636
-1,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333
-1,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636
-1,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636
-2,Partnership Agreement,0.516078,0.027879,0.440784,0.032424
-2,SisterCities,0.169804,0.033333,0.238824,0.028182
-2,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788
-2,Toolkit,0.83098,0.072727,0.127059,0.025455
-2,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303
-2,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333
-2,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333
-2,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333
-2,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636
-2,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333
-2,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333
-2,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333
-2,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333
-2,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636
-2,General Guidelines,0.118039,0.295152,0.231765,0.016061
-2,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636
-2,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636
-2,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636
-2,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636
-2,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636
-2,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333
-2,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636
-2,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636
-2,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939
-2,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636
-2,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636
-2,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636
-2,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939
-2,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333
-2,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636
-2,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636
-2,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939
-2,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636
-2,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939
-2,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636
-2,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333
-2,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636
-2,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636
-2,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636
-2,with very specific tasks.,0.176078,0.750606,0.190196,0.013333
-2,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636
-2,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333
-2,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939
-2,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636
-2,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939
-2,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636
-3,Partnership Agreement,0.516078,0.027879,0.441176,0.032121
-3,SisterCities,0.169804,0.033333,0.239216,0.028182
-3,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788
-3,Toolkit,0.83098,0.07303,0.126667,0.025152
-3,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303
-3,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333
-3,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333
-3,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636
-3,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636
-3,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636
-3,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333
-3,and cooperation.,0.176471,0.25697,0.13451,0.013333
-3,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333
-3,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636
-3,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333
-3,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636
-3,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636
-3,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636
-3,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636
-3,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333
-3,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636
-3,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333
-3,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636
-3,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636
-3,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636
-3,for their records.,0.176078,0.550606,0.131373,0.010606
-3,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636
-3,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636
-3,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333
-3,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939
-3,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636
-3,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242
-3,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515
-3,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939
-3,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636
-3,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636
-3,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636
-3,sending it to our Membership Director at [email protected] or contacting us at (202),0.117647,0.782727,0.732157,0.013636
-3,347-8630.,0.117647,0.799394,0.080392,0.010303
-4,Partnership Agreement,0.516471,0.027879,0.440784,0.032727
-4,SisterCities,0.169412,0.033333,0.239608,0.028485
-4,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091
-4,Toolkit,0.830588,0.072727,0.127843,0.025758
-4,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333
-4,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394
-4,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667
-4,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727
-4,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121
-4,BETWEEN,0.454902,0.413636,0.110588,0.011212
-4,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939
-4,AND,0.487843,0.452727,0.048235,0.011212
-4,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848
-4,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303
-4,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727
-4,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727
-4,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424
-4,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424
-4,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303
-4,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121
-4,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424
-4,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121
-4,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758
-4,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303
-4,relationship became effective.,0.221569,0.705152,0.217647,0.012424
-4,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303
-4,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727
-4,A,0.344314,0.768485,0.084706,0.030303
-4,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909
-4,Lee P.Brown,0.729412,0.806364,0.118824,0.010303
-4,Mayor of Houston,0.704706,0.823333,0.166667,0.012424
-4,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727
-4,&Town Planning,0.324314,0.841212,0.155686,0.012424
-5,Partnership Agreement,0.516078,0.027879,0.441176,0.032424
-5,SisterCities,0.169412,0.033333,0.239608,0.028485
-5,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091
-5,Toolkit,0.83098,0.072727,0.127059,0.025758
-5,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333
-5,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697
-5,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697
-5,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303
-5,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818
-5,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333
-5,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242
-5,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636
-5,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152
-5,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455
-5,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152
-5,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152
-5,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455
-5,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848
-5,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939
-5,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545
-5,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848
-5,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152
-5,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242
-5,the cities;,0.22902,0.624545,0.076471,0.012424
-5,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152
-5,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636
-5,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636
-5,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848
-5,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545
-5,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152
-5,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545
-5,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909
-5,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636
-5,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091
-5,Mayor,0.311373,0.894848,0.053333,0.012727
-5,New York City,0.287843,0.909091,0.121176,0.013333
-5,London,0.701961,0.909091,0.061569,0.010606
-6,Partnership Agreement,0.515686,0.027576,0.441961,0.03303
-6,SisterCities,0.169412,0.03303,0.24,0.028182
-6,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091
-6,Toolkit,0.83098,0.072727,0.127451,0.025758
-6,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333
-6,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364
-6,City of Long Beach,0.388627,0.196667,0.476471,0.066364
-6,California,0.551373,0.257273,0.136471,0.033333
-6,Sister City Agreement,0.321961,0.305455,0.378431,0.035152
-6,between the,0.464706,0.352727,0.084314,0.009697
-6,City of Long Beach,0.38,0.378485,0.252549,0.01697
-6,"California, USA",0.4,0.397576,0.21098,0.016061
-6,and the,0.48,0.415152,0.053333,0.009091
-6,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697
-6,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152
-6,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121
-6,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303
-6,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121
-6,purposes:,0.216863,0.516061,0.058039,0.009394
-6,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424
-6,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424
-6,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424
-6,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121
-6,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121
-6,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121
-6,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727
-6,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697
-6,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727
-6,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424
-6,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121
-6,STATE OFFICE,0.276471,0.713636,0.050588,0.048788
-6,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636
-6,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636
-6,"California, USA",0.582745,0.765758,0.125098,0.01303
-6,10.2aulus,0.490588,0.771818,0.220392,0.062424
-6,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333
-6,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636
-6,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818
-6,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303
-7,Partnership Agreement,0.516078,0.027879,0.441176,0.032424
-7,SisterCities,0.169412,0.03303,0.24,0.028485
-7,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091
-7,Toolkit,0.83098,0.072727,0.127451,0.025758
-7,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333
-7,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939
-7,adopted by,0.2,0.213333,0.080392,0.013636
-7,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424
-7,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515
-7,and,0.199608,0.260909,0.026275,0.010606
-7,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212
-7,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212
-7,ON,0.551765,0.298182,0.026667,0.011515
-7,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848
-7,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152
-7,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455
-7,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848
-7,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242
-7,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848
-7,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242
-7,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152
-7,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242
-7,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242
-7,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455
-7,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636
-7,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455
-7,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606
-7,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758
-7,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242
-7,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848
-7,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152
-7,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636
-7,3h.5.,0.593725,0.750606,0.218039,0.06303
-7,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818
-7,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606
-7,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303
-7,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606
+page,text,left,top,width,height,line
+1,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,1
+1,SisterCities,0.169804,0.033333,0.238431,0.028182,2
+1,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+1,Toolkit,0.830588,0.07303,0.126667,0.025152,4
+1,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303,5
+1,Types of Affiliations,0.117255,0.157576,0.241961,0.02,6
+1,Sister City Relationship,0.117647,0.187273,0.196863,0.013939,7
+1,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636,8
+1,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939,9
+1,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636,10
+1,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939,11
+1,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636,12
+1,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636,13
+1,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636,14
+1,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636,15
+1,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636,16
+1,Friendship City,0.118039,0.372121,0.127059,0.013939,17
+1,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636,18
+1,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242,19
+1,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636,20
+1,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636,21
+1,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636,22
+1,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333,23
+1,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636,24
+1,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636,25
+1,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636,26
+1,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333,27
+1,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727,28
+1,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636,29
+1,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333,30
+1,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939,31
+1,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242,32
+1,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636,33
+1,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636,34
+1,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939,35
+1,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939,36
+1,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636,37
+1,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939,38
+1,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242,39
+1,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333,40
+1,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939,41
+1,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636,42
+1,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333,43
+1,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636,44
+1,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636,45
+2,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,1
+2,SisterCities,0.169804,0.033333,0.238824,0.028182,2
+2,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+2,Toolkit,0.83098,0.072727,0.127059,0.025455,4
+2,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,5
+2,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333,6
+2,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333,7
+2,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333,8
+2,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636,9
+2,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333,10
+2,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333,11
+2,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333,12
+2,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333,13
+2,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636,14
+2,General Guidelines,0.118039,0.295152,0.231765,0.016061,15
+2,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636,16
+2,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636,17
+2,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636,18
+2,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636,19
+2,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636,20
+2,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333,21
+2,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636,22
+2,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636,23
+2,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939,24
+2,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636,25
+2,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636,26
+2,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636,27
+2,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939,28
+2,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333,29
+2,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636,30
+2,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636,31
+2,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939,32
+2,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636,33
+2,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939,34
+2,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636,35
+2,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333,36
+2,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636,37
+2,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636,38
+2,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636,39
+2,with very specific tasks.,0.176078,0.750606,0.190196,0.013333,40
+2,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636,41
+2,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333,42
+2,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939,43
+2,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636,44
+2,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939,1
+2,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636,2
+3,Partnership Agreement,0.516078,0.027879,0.441176,0.032121,3
+3,SisterCities,0.169804,0.033333,0.239216,0.028182,4
+3,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788,5
+3,Toolkit,0.83098,0.07303,0.126667,0.025152,6
+3,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,7
+3,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333,8
+3,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333,9
+3,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636,10
+3,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636,11
+3,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636,12
+3,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333,13
+3,and cooperation.,0.176471,0.25697,0.13451,0.013333,14
+3,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333,15
+3,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636,16
+3,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333,17
+3,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636,18
+3,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636,19
+3,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636,20
+3,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636,21
+3,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333,22
+3,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636,23
+3,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333,24
+3,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636,25
+3,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636,26
+3,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636,27
+3,for their records.,0.176078,0.550606,0.131373,0.010606,28
+3,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636,29
+3,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636,30
+3,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333,31
+3,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939,32
+3,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636,33
+3,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242,34
+3,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515,35
+3,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939,36
+3,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636,37
+3,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636,38
+3,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636,39
+3,sending it to our Membership Director at [email protected] or contacting us at (202),0.117647,0.782727,0.732157,0.013636,40
+3,347-8630.,0.117647,0.799394,0.080392,0.010303,41
+4,Partnership Agreement,0.516471,0.027879,0.440784,0.032727,1
+4,SisterCities,0.169412,0.033333,0.239608,0.028485,2
+4,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091,3
+4,Toolkit,0.830588,0.072727,0.127843,0.025758,4
+4,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,5
+4,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394,6
+4,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667,7
+4,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727,8
+4,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121,9
+4,BETWEEN,0.454902,0.413636,0.110588,0.011212,10
+4,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939,11
+4,AND,0.487843,0.452727,0.048235,0.011212,12
+4,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848,13
+4,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303,14
+4,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727,15
+4,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727,16
+4,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424,17
+4,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424,18
+4,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303,19
+4,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121,20
+4,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424,21
+4,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121,22
+4,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758,23
+4,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303,24
+4,relationship became effective.,0.221569,0.705152,0.217647,0.012424,25
+4,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303,26
+4,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727,27
+4,A,0.344314,0.768485,0.084706,0.030303,28
+4,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909,29
+4,Lee P.Brown,0.729412,0.806364,0.118824,0.010303,30
+4,Mayor of Houston,0.704706,0.823333,0.166667,0.012424,31
+4,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727,32
+4,&Town Planning,0.324314,0.841212,0.155686,0.012424,33
+5,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,1
+5,SisterCities,0.169412,0.033333,0.239608,0.028485,2
+5,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091,3
+5,Toolkit,0.83098,0.072727,0.127059,0.025758,4
+5,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,5
+5,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697,6
+5,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697,7
+5,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303,8
+5,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818,9
+5,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333,10
+5,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242,11
+5,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636,12
+5,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152,13
+5,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455,14
+5,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152,15
+5,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152,16
+5,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455,17
+5,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848,18
+5,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939,19
+5,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545,20
+5,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848,21
+5,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152,22
+5,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242,23
+5,the cities;,0.22902,0.624545,0.076471,0.012424,24
+5,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152,25
+5,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636,26
+5,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636,27
+5,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848,28
+5,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545,29
+5,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152,30
+5,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545,31
+5,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909,32
+5,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636,33
+5,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091,34
+5,Mayor,0.311373,0.894848,0.053333,0.012727,35
+5,New York City,0.287843,0.909091,0.121176,0.013333,36
+5,London,0.701961,0.909091,0.061569,0.010606,37
+6,Partnership Agreement,0.515686,0.027576,0.441961,0.03303,1
+6,SisterCities,0.169412,0.03303,0.24,0.028182,2
+6,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091,3
+6,Toolkit,0.83098,0.072727,0.127451,0.025758,4
+6,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,5
+6,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364,6
+6,City of Long Beach,0.388627,0.196667,0.476471,0.066364,7
+6,California,0.551373,0.257273,0.136471,0.033333,8
+6,Sister City Agreement,0.321961,0.305455,0.378431,0.035152,9
+6,between the,0.464706,0.352727,0.084314,0.009697,10
+6,City of Long Beach,0.38,0.378485,0.252549,0.01697,11
+6,"California, USA",0.4,0.397576,0.21098,0.016061,12
+6,and the,0.48,0.415152,0.053333,0.009091,13
+6,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697,14
+6,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152,15
+6,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121,16
+6,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303,17
+6,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121,18
+6,purposes:,0.216863,0.516061,0.058039,0.009394,19
+6,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424,20
+6,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424,21
+6,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424,22
+6,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121,23
+6,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121,24
+6,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121,25
+6,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727,26
+6,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697,27
+6,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727,28
+6,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424,29
+6,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121,30
+6,STATE OFFICE,0.276471,0.713636,0.050588,0.048788,31
+6,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636,32
+6,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636,33
+6,"California, USA",0.582745,0.765758,0.125098,0.01303,34
+6,10.2aulus,0.490588,0.771818,0.220392,0.062424,35
+6,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333,36
+6,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636,37
+6,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818,38
+6,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303,39
+7,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,1
+7,SisterCities,0.169412,0.03303,0.24,0.028485,2
+7,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091,3
+7,Toolkit,0.83098,0.072727,0.127451,0.025758,4
+7,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,5
+7,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939,6
+7,adopted by,0.2,0.213333,0.080392,0.013636,7
+7,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424,8
+7,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515,9
+7,and,0.199608,0.260909,0.026275,0.010606,10
+7,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212,11
+7,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212,12
+7,ON,0.551765,0.298182,0.026667,0.011515,13
+7,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848,14
+7,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152,15
+7,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455,16
+7,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848,17
+7,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242,18
+7,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848,19
+7,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242,20
+7,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152,21
+7,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242,22
+7,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242,23
+7,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455,24
+7,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636,25
+7,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455,26
+7,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606,27
+7,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758,28
+7,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242,29
+7,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848,30
+7,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152,31
+7,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636,32
+7,3h.5.,0.593725,0.750606,0.218039,0.06303,33
+7,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818,34
+7,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606,35
+7,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303,36
+7,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606,37

tools/config.py

@@ -161,11 +161,9 @@ if OUTPUT_FOLDER == "TEMP" or INPUT_FOLDER == "TEMP":
             INPUT_FOLDER = temp_dir + "/"
 
 GRADIO_TEMP_DIR = get_or_create_env_var(
-    "GRADIO_TEMP_DIR", "tmp/gradio_tmp/"
+    "GRADIO_TEMP_DIR", ""
 )  # Default Gradio temp folder
-MPLCONFIGDIR = get_or_create_env_var(
-    "MPLCONFIGDIR", "tmp/matplotlib_cache/"
-)  # Matplotlib cache folder
+MPLCONFIGDIR = get_or_create_env_var("MPLCONFIGDIR", "")  # Matplotlib cache folder
 
 ###
 # LOGGING OPTIONS
@@ -545,6 +543,8 @@ except Exception as e:
 # Get some environment variables and Launch the Gradio app
 COGNITO_AUTH = get_or_create_env_var("COGNITO_AUTH", "0")
 
+SHOW_EXAMPLES = get_or_create_env_var("SHOW_EXAMPLES", "True")
+
 RUN_DIRECT_MODE = get_or_create_env_var("RUN_DIRECT_MODE", "0")
 
 # Direct mode configuration options
@@ -574,7 +574,7 @@ ROOT_PATH = get_or_create_env_var("ROOT_PATH", "")
 
 DEFAULT_CONCURRENCY_LIMIT = int(get_or_create_env_var("DEFAULT_CONCURRENCY_LIMIT", "3"))
 
-FILE_INPUT_HEIGHT = get_or_create_env_var("FILE_INPUT_HEIGHT", "200")
+FILE_INPUT_HEIGHT = int(get_or_create_env_var("FILE_INPUT_HEIGHT", "200"))
 
 ### ALLOW LIST
 

tools/data_anonymise.py

@@ -515,8 +515,6 @@ def anonymise_files_with_open_text(
     if isinstance(out_message, str):
         out_message = [out_message]
 
-    # print("log_files_output_paths:",log_files_output_paths)
-
     if isinstance(log_files_output_paths, str):
         log_files_output_paths = list()
 

tools/file_conversion.py

@@ -87,9 +87,6 @@ def is_pdf(filename):
     return filename.lower().endswith(".pdf")
 
 
-## Convert pdf to image if necessary
-
-
 def check_image_size_and_reduce(out_path: str, image: Image):
     """
     Check if a given image size is above around 4.5mb, and reduce size if necessary. 5mb is the maximum possible to submit to AWS Textract.
@@ -297,7 +294,6 @@ def process_file_for_image_creation(
 
     # Check if the file is a PDF
     elif file_extension == ".pdf":
-        # print(f"{file_path} is a PDF file. Converting to image set")
 
         # Run your function for processing PDF files here
         img_path, image_sizes_width, image_sizes_height, all_img_details = (
@@ -653,8 +649,8 @@ def word_level_ocr_output_to_dataframe(ocr_results: dict) -> pd.DataFrame:
 def prepare_image_or_pdf(
     file_paths: List[str],
     text_extract_method: str,
-    all_line_level_ocr_results_df: pd.DataFrame,
-    all_page_line_level_ocr_results_with_words_df: pd.DataFrame,
+    all_line_level_ocr_results_df: pd.DataFrame = None,
+    all_page_line_level_ocr_results_with_words_df: pd.DataFrame = None,
     latest_file_completed: int = 0,
     out_message: List[str] = list(),
     first_loop_state: bool = False,