"""Normalize the spelling of the data used for machine learning

Usage: python normalize_spelling <train set> <test set> <output dir>
"""

import codecs
import json
import argparse
from collections import Counter
import os
import string


with codecs.open('hist2modern_liwc.json', 'rb', 'utf-8') as f:
    full_dict = json.load(f, 'utf-8')

# create simple historic word -> modern word mapping
# (full_dict may contain multiple alternatives for a word)
hist2modern = {}
for w in full_dict.keys():
    if w not in full_dict[w]:
        c = Counter(full_dict[w])
        hist2modern[w] = c.most_common()[0][0]
print '#words in dict: {}'.format(len(hist2modern))

parser = argparse.ArgumentParser()
parser.add_argument('train', help='the name of the file containing the train '
                    'data.')
parser.add_argument('test', help='the name of the file containing the test '
                    'data.')
parser.add_argument('output_dir', help='the directory where the '
                    'generated text files should be saved')
args = parser.parse_args()

data_files = [args.train, args.test]

num_words = 0
num_replaced = 0
replacements = Counter()

for df in data_files:
    new_lines = []
    with codecs.open(df, 'rb', 'utf-8') as f:
        for line in f.readlines():
            l, label = line.rsplit(None, 1)
            words = l.split()
            result = []
            for word in words:
                w = word.lower()

                if w not in string.punctuation:
                    num_words += 1

                if w in hist2modern.keys():
                    result.append(hist2modern[w])
                    num_replaced += 1
                    replacements[w] += 1
                else:
                    result.append(word)
            new_lines.append('{}\t{}'.format(' '.join(result).encode('utf-8'),
                                             label))

    # write output
    _head, tail = os.path.split(df)
    out_file_name = '{}-normalized_spelling.txt'.format(tail.split('.')[0])
    out_file = os.path.join(args.output_dir, out_file_name)
    with codecs.open(out_file, 'wb', 'utf-8') as f:
        f.write('\n'.join(new_lines).decode('utf-8'))

# print number of replacements
print 'total words\t{}\ntotal replaced\t{}'.format(num_words, num_replaced)
for replaced, freq in replacements.most_common():
    print '{}\t{}\t{}'.format(replaced.encode('utf-8'),
                              hist2modern[replaced].encode('utf-8'),
                              freq)

Add script to normalize spelling
Add script to normalize spelling

Added a script that takes as input a training and test set containing
sentences with words separated by spaces, normalizes the spelling using
the hist2modern dictionary and writes files containing the spelling
normalized train and test set. It also outputs to std out the counts of
words replaced.
