# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:ocr] *
#     language: python
#     name: conda-env-ocr-py
# ---

# In this notebook, I am applying some standard text cleaning steps to the text files. These are steps that would be applied in any standard situation and so are necessary to avoid "stacking the deck" against the OCR-generated text.
#
# These are:
# - removing special characters
# - reconnecting line endings (changed mind on this idea).
# - remove single character words (excluding "a" and "i")
#

# +
import os
import re

project_directory = "/Users/jeriwieringa/Documents/Research/ocr-and-nlp/"
input_directory = "data/text/ocr-generated/"
output_directory = "data/text/corrected"
# -

file_list = os.listdir(os.path.join(project_directory, input_directory)) 

file_list[:5]


def open_and_read(file_name):
    with open(os.path.join(project_directory, input_directory, file_name), 'r') as f:
        text_blob = f.read()
    
    return text_blob


def remove_special_characters(text_blob):
#     text_blob = re.sub("[^!,.:;-?](?= |$)", "", text_blob)
    text_blob = re.sub("[^.,?!a-zA-Z0-9 ]", "", text_blob)
#     text_blob = re.sub("[^\w]", "", text_blob)
#     text_blob = re.sub("[\n\r]", " ", text_blob)
    
    return text_blob


def remove_single_character_words(text_blob):
    "Not working"
    cleaned = re.sub("(?<=(^|\s))[^aiAI](\s|$)", " ", text_blob)
    return cleaned


def write_to_file(file_name, text_blob):
    with open(os.path.join(project_directory, output_directory, file_name), 'w') as o:
        o.write(text_blob)
    
    return


for file_name in file_list:
    if not file_name.startswith('.'):
        contents = open_and_read(file_name)
        cleaned_contents = remove_special_characters(contents)
#         cleaned_contents = remove_single_character_words(cleaned_contents)
        write_to_file(file_name, cleaned_contents)







StarcoderdataJupyterScriptsDedupFiltered