tasks/vowels/preprocess.py
def main():
    """
    Generate the data used for training the DNC on how to find vowels in words. Create data directories storing this
    information in its unencoded form and encoded form.
    :return: None.
    """
    task_dir = dirname(abspath(__file__))
    options, _ = getopt.getopt(sys.argv[1:], '', ['data_dir=', 'single_train'])
    joint_train = True
    files_list = []
    total_examples = 10000

    if not exists(join(task_dir, 'data')):
        mkdir(join(task_dir, 'data'))
    if not exists(join(task_dir, 'data', 'unencoded')):
        mkdir(join(task_dir, 'data', 'unencoded'))
    generate_data(join(task_dir, 'data', 'unencoded'), total_examples)
    data_dir = join(task_dir, 'data', 'unencoded')

    for opt in options:
        if opt[0] == '--single_train':
            joint_train = False

    for entry_name in listdir(data_dir):
        entry_path = join(data_dir, entry_name)
        if isfile(entry_path) and entry_path.endswith('txt'):
            files_list.append(entry_path)

    lexicon_dictionary = create_dictionary(files_list)
    lexicon_count = len(lexicon_dictionary)

    encoded_files = encode_data(files_list, lexicon_dictionary)

    print(("Total Number of examples: %d" % (len(encoded_files))))

    processed_data_dir = join(task_dir, 'data', 'encoded')
    train_data_dir = join(processed_data_dir, 'train')
    test_data_dir = join(processed_data_dir, 'test')
    if exists(processed_data_dir) and isdir(processed_data_dir):
        rmtree(processed_data_dir)

    mkdir(processed_data_dir)
    mkdir(train_data_dir)
    mkdir(test_data_dir)

    llprint("Saving processed data to disk ... ")

    pickle.dump(lexicon_dictionary, open(join(processed_data_dir, 'lexicon-dict.pkl'), 'wb'))

    joint_train_data = []

    for filename in encoded_files:
        if filename.endswith("test.txt"):
            pickle.dump(encoded_files[filename], open(join(test_data_dir, basename(filename) + '.pkl'), 'wb'))
        elif filename.endswith("train.txt"):
            if not joint_train:
                pickle.dump(encoded_files[filename], open(join(train_data_dir, basename(filename) + '.pkl'), 'wb'))
            else:
                joint_train_data.extend(encoded_files[filename])

    if joint_train:

        pickle.dump(joint_train_data, open(join(train_data_dir, 'train.pkl'), 'wb'))

    llprint("Done!\n")

    Generate the data used for training the DNC on how to find vowels in words. Create data directories storing this
    information in its unencoded form and encoded form.
    :return: None.
    
Generate the data used for training the DNC on how to find vowels in words. Create data directories storing this
information in its unencoded form and encoded form.
Generate the data used for training the DNC on how to find vowels in words . Create data directories storing this information in its unencoded form and encoded form .
def main():
    task_dir = dirname(abspath(__file__))
    options, _ = getopt.getopt(sys.argv[1:], '', ['data_dir=', 'single_train'])
    joint_train = True
    files_list = []
    total_examples = 10000
    if not exists(join(task_dir, 'data')):
        mkdir(join(task_dir, 'data'))
    if not exists(join(task_dir, 'data', 'unencoded')):
        mkdir(join(task_dir, 'data', 'unencoded'))
    generate_data(join(task_dir, 'data', 'unencoded'), total_examples)
    data_dir = join(task_dir, 'data', 'unencoded')
    for opt in options:
        if opt[0] == '--single_train':
            joint_train = False
    for entry_name in listdir(data_dir):
        entry_path = join(data_dir, entry_name)
        if isfile(entry_path) and entry_path.endswith('txt'):
            files_list.append(entry_path)
    lexicon_dictionary = create_dictionary(files_list)
    lexicon_count = len(lexicon_dictionary)
    encoded_files = encode_data(files_list, lexicon_dictionary)
    print(("Total Number of examples: %d" % (len(encoded_files))))
    processed_data_dir = join(task_dir, 'data', 'encoded')
    train_data_dir = join(processed_data_dir, 'train')
    test_data_dir = join(processed_data_dir, 'test')
    if exists(processed_data_dir) and isdir(processed_data_dir):
        rmtree(processed_data_dir)
    mkdir(processed_data_dir)
    mkdir(train_data_dir)
    mkdir(test_data_dir)
    llprint("Saving processed data to disk ... ")
    pickle.dump(lexicon_dictionary, open(join(processed_data_dir, 'lexicon-dict.pkl'), 'wb'))
    joint_train_data = []
    for filename in encoded_files:
        if filename.endswith("test.txt"):
            pickle.dump(encoded_files[filename], open(join(test_data_dir, basename(filename) + '.pkl'), 'wb'))
        elif filename.endswith("train.txt"):
            if not joint_train:
                pickle.dump(encoded_files[filename], open(join(train_data_dir, basename(filename) + '.pkl'), 'wb'))
            else:
                joint_train_data.extend(encoded_files[filename])
    if joint_train:
        pickle.dump(joint_train_data, open(join(train_data_dir, 'train.pkl'), 'wb'))
    llprint("Done!\n")
def main ( ) : task_dir = dirname ( abspath ( __file__ ) ) options , _ = getopt . getopt ( sys . argv [ 1 : ] , '' , [ 'data_dir=' , 'single_train' ] ) joint_train = True files_list = [ ] total_examples = 10000 if not exists ( join ( task_dir , 'data' ) ) : mkdir ( join ( task_dir , 'data' ) ) if not exists ( join ( task_dir , 'data' , 'unencoded' ) ) : mkdir ( join ( task_dir , 'data' , 'unencoded' ) ) generate_data ( join ( task_dir , 'data' , 'unencoded' ) , total_examples ) data_dir = join ( task_dir , 'data' , 'unencoded' ) for opt in options : if opt [ 0 ] == '--single_train' : joint_train = False for entry_name in listdir ( data_dir ) : entry_path = join ( data_dir , entry_name ) if isfile ( entry_path ) and entry_path . endswith ( 'txt' ) : files_list . append ( entry_path ) lexicon_dictionary = create_dictionary ( files_list ) lexicon_count = len ( lexicon_dictionary ) encoded_files = encode_data ( files_list , lexicon_dictionary ) print ( ( "Total Number of examples: %d" % ( len ( encoded_files ) ) ) ) processed_data_dir = join ( task_dir , 'data' , 'encoded' ) train_data_dir = join ( processed_data_dir , 'train' ) test_data_dir = join ( processed_data_dir , 'test' ) if exists ( processed_data_dir ) and isdir ( processed_data_dir ) : rmtree ( processed_data_dir ) mkdir ( processed_data_dir ) mkdir ( train_data_dir ) mkdir ( test_data_dir ) llprint ( "Saving processed data to disk ... " ) pickle . dump ( lexicon_dictionary , open ( join ( processed_data_dir , 'lexicon-dict.pkl' ) , 'wb' ) ) joint_train_data = [ ] for filename in encoded_files : if filename . endswith ( "test.txt" ) : pickle . dump ( encoded_files [ filename ] , open ( join ( test_data_dir , basename ( filename ) + '.pkl' ) , 'wb' ) ) elif filename . endswith ( "train.txt" ) : if not joint_train : pickle . dump ( encoded_files [ filename ] , open ( join ( train_data_dir , basename ( filename ) + '.pkl' ) , 'wb' ) ) else : joint_train_data . extend ( encoded_files [ filename ] ) if joint_train : pickle . dump ( joint_train_data , open ( join ( train_data_dir , 'train.pkl' ) , 'wb' ) ) llprint ( "Done!\n" )
Generate the data used for training the DNC on how to find vowels in words.
Generate the data used for training the DNC on how to find vowels in words .
"""
    Generate the data used for training the DNC on how to find vowels in words. Create data directories storing this
    information in its unencoded form and encoded form.
    :return: None.
    """
None