tasks/DREAM/preprocess.py
def main():
    """
    Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory
    that contains the same files. But in these files, the dialogue, question and answer's words are cleaned. A second
    new directory is also created, this directory stores the cleaned data in its numerical format.
    their numerical equivalent. If single_train is specified, all training and testing dialogues are dumped into the
    same json file. Otherwise, training and testing data is stored separately.
    :return:
    """
    task_dir = dirname(abspath(__file__))
    options, _ = getopt.getopt(sys.argv[1:], '', ['data_dir=', 'single_train', 'length_limit='])
    data_dir = None
    joint_train = True
    length_limit = None
    training_files = []
    testing_files = []

    if not exists(join(task_dir, 'data')):
        mkdir(join(task_dir, 'data'))
    if not exists(join(task_dir, 'data', 'clean')):
        mkdir(join(task_dir, 'data', 'clean'))

    for opt in options:
        if opt[0] == '--data_dir':
            data_dir = opt[1]
        if opt[0] == '--single_train':
            joint_train = False
    if not exists(join(task_dir, 'data')):
        mkdir(join(task_dir, 'data'))
        if opt[0] == '--length_limit':
            length_limit = int(opt[1])

    if data_dir is None:
        if exists(join(task_dir, 'data', 'unencoded')):
            data_dir = join(task_dir, 'data', 'unencoded')
        else:
            raise ValueError("data_dir argument cannot be None")

    all_questions = []
    for a_file in listdir(data_dir):
        if a_file.endswith(".json"):
            full_path = join(data_dir, a_file)
            with open(full_path) as json_file:
                a_list = json.load(json_file)
                a_list = clean_data(a_list)
                for i, item in enumerate(a_list):
                    write_path = join(task_dir, 'data', 'clean', f"{item[2].replace(' ', '')}_{a_file}")
                    with open(write_path, 'w+') as write_file:
                        json.dump(item, write_file)
                    if a_file.endswith("train.json"):
                        training_files.append(write_path)
                    else:
                        testing_files.append(write_path)

                all_questions.extend(a_list)

    lexicon_dictionary = create_dictionary(all_questions)

    print(f"There are {len(lexicon_dictionary)} unique words; "
          f"{len([entry for entry in lexicon_dictionary if entry.isnumeric()])} of these words are numbers, "
          f"{len([entry for entry in lexicon_dictionary if entry.isalpha()])} of these words are standard alphabetical"
          f" words, and "
          f"{len([entry for entry in lexicon_dictionary if entry in punctuation])} of these words are punctuation "
          f"marks.")
    print([entry for entry in lexicon_dictionary if not entry.isnumeric() and not entry.isalpha() and entry not in
           punctuation])
    with open(join(task_dir, 'data', 'dictionary_entries.json'), 'w+') as write_file:
        json.dump(list(lexicon_dictionary.keys()), write_file)

    processed_data_dir = join(task_dir, 'data', 'encoded')
    train_data_dir = join(processed_data_dir, 'train')
    test_data_dir = join(processed_data_dir, 'test')
    if exists(processed_data_dir) and isdir(processed_data_dir):
        rmtree(processed_data_dir)

    mkdir(processed_data_dir)
    mkdir(train_data_dir)
    mkdir(test_data_dir)

    encoded_training_files, training_stories_lengths = encode_data(training_files, train_data_dir, lexicon_dictionary)
    encoded_testing_files, testing_stories_lengths = encode_data(testing_files, test_data_dir, lexicon_dictionary)
    encoded_files = encoded_training_files + encoded_testing_files

    stories_lengths = np.array(training_stories_lengths + testing_stories_lengths)
    length_limit = np.max(stories_lengths) if length_limit is None else length_limit
    print("Total Number of stories: %d" % (len(stories_lengths)))
    print("Number of stories with lengthens > %d: %d (%% %.2f) [discarded]" % (length_limit,
                                                                               int(np.sum(
                                                                                   stories_lengths > length_limit)),
                                                                               np.mean(stories_lengths >
                                                                                       length_limit) * 100.0))
    print("Number of Remaining Stories: %d" % (len(stories_lengths[stories_lengths <= length_limit])))

    llprint("Saving processed data to disk ... ")

    pickle.dump(lexicon_dictionary, open(join(processed_data_dir, 'lexicon-dict.pkl'), 'wb'))

    joint_train_data = []

    for filename in encoded_files:
        if filename.endswith("test.json"):
            pickle.dump(encoded_files[filename], open(join(test_data_dir, basename(filename) + '.pkl'), 'wb'))
        elif filename.endswith("train.json"):
            if not joint_train:
                pickle.dump(encoded_files[filename], open(join(train_data_dir, basename(filename) + '.pkl'), 'wb'))
            else:
                joint_train_data.extend(encoded_files[filename])

    if joint_train:
        pickle.dump(joint_train_data, open(join(train_data_dir, 'train.pkl'), 'wb'))

    llprint("Done!\n")

    Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory
    that contains the same files. But in these files, the dialogue, question and answer's words are cleaned. A second
    new directory is also created, this directory stores the cleaned data in its numerical format.
    their numerical equivalent. If single_train is specified, all training and testing dialogues are dumped into the
    same json file. Otherwise, training and testing data is stored separately.
    :return:
    
Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory
that contains the same files. But in these files, the dialogue, question and answer's words are cleaned. A second
new directory is also created, this directory stores the cleaned data in its numerical format.
their numerical equivalent. If single_train is specified, all training and testing dialogues are dumped into the
same json file. Otherwise, training and testing data is stored separately.
Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory that contains the same files . But in these files the dialogue question and answer ' s words are cleaned . A second new directory is also created this directory stores the cleaned data in its numerical format . their numerical equivalent . If single_train is specified all training and testing dialogues are dumped into the same json file . Otherwise training and testing data is stored separately .
def main():
    task_dir = dirname(abspath(__file__))
    options, _ = getopt.getopt(sys.argv[1:], '', ['data_dir=', 'single_train', 'length_limit='])
    data_dir = None
    joint_train = True
    length_limit = None
    training_files = []
    testing_files = []
    if not exists(join(task_dir, 'data')):
        mkdir(join(task_dir, 'data'))
    if not exists(join(task_dir, 'data', 'clean')):
        mkdir(join(task_dir, 'data', 'clean'))
    for opt in options:
        if opt[0] == '--data_dir':
            data_dir = opt[1]
        if opt[0] == '--single_train':
            joint_train = False
    if not exists(join(task_dir, 'data')):
        mkdir(join(task_dir, 'data'))
        if opt[0] == '--length_limit':
            length_limit = int(opt[1])
    if data_dir is None:
        if exists(join(task_dir, 'data', 'unencoded')):
            data_dir = join(task_dir, 'data', 'unencoded')
        else:
            raise ValueError("data_dir argument cannot be None")
    all_questions = []
    for a_file in listdir(data_dir):
        if a_file.endswith(".json"):
            full_path = join(data_dir, a_file)
            with open(full_path) as json_file:
                a_list = json.load(json_file)
                a_list = clean_data(a_list)
                for i, item in enumerate(a_list):
                    write_path = join(task_dir, 'data', 'clean', f"{item[2].replace(' ', '')}_{a_file}")
                    with open(write_path, 'w+') as write_file:
                        json.dump(item, write_file)
                    if a_file.endswith("train.json"):
                        training_files.append(write_path)
                    else:
                        testing_files.append(write_path)
                all_questions.extend(a_list)
    lexicon_dictionary = create_dictionary(all_questions)
    print(f"There are {len(lexicon_dictionary)} unique words; "
          f"{len([entry for entry in lexicon_dictionary if entry.isnumeric()])} of these words are numbers, "
          f"{len([entry for entry in lexicon_dictionary if entry.isalpha()])} of these words are standard alphabetical"
          f" words, and "
          f"{len([entry for entry in lexicon_dictionary if entry in punctuation])} of these words are punctuation "
          f"marks.")
    print([entry for entry in lexicon_dictionary if not entry.isnumeric() and not entry.isalpha() and entry not in
           punctuation])
    with open(join(task_dir, 'data', 'dictionary_entries.json'), 'w+') as write_file:
        json.dump(list(lexicon_dictionary.keys()), write_file)
    processed_data_dir = join(task_dir, 'data', 'encoded')
    train_data_dir = join(processed_data_dir, 'train')
    test_data_dir = join(processed_data_dir, 'test')
    if exists(processed_data_dir) and isdir(processed_data_dir):
        rmtree(processed_data_dir)
    mkdir(processed_data_dir)
    mkdir(train_data_dir)
    mkdir(test_data_dir)
    encoded_training_files, training_stories_lengths = encode_data(training_files, train_data_dir, lexicon_dictionary)
    encoded_testing_files, testing_stories_lengths = encode_data(testing_files, test_data_dir, lexicon_dictionary)
    encoded_files = encoded_training_files + encoded_testing_files
    stories_lengths = np.array(training_stories_lengths + testing_stories_lengths)
    length_limit = np.max(stories_lengths) if length_limit is None else length_limit
    print("Total Number of stories: %d" % (len(stories_lengths)))
    print("Number of stories with lengthens > %d: %d (%% %.2f) [discarded]" % (length_limit,
                                                                               int(np.sum(
                                                                                   stories_lengths > length_limit)),
                                                                               np.mean(stories_lengths >
                                                                                       length_limit) * 100.0))
    print("Number of Remaining Stories: %d" % (len(stories_lengths[stories_lengths <= length_limit])))
    llprint("Saving processed data to disk ... ")
    pickle.dump(lexicon_dictionary, open(join(processed_data_dir, 'lexicon-dict.pkl'), 'wb'))
    joint_train_data = []
    for filename in encoded_files:
        if filename.endswith("test.json"):
            pickle.dump(encoded_files[filename], open(join(test_data_dir, basename(filename) + '.pkl'), 'wb'))
        elif filename.endswith("train.json"):
            if not joint_train:
                pickle.dump(encoded_files[filename], open(join(train_data_dir, basename(filename) + '.pkl'), 'wb'))
            else:
                joint_train_data.extend(encoded_files[filename])
    if joint_train:
        pickle.dump(joint_train_data, open(join(train_data_dir, 'train.pkl'), 'wb'))
    llprint("Done!\n")
def main ( ) : task_dir = dirname ( abspath ( __file__ ) ) options , _ = getopt . getopt ( sys . argv [ 1 : ] , '' , [ 'data_dir=' , 'single_train' , 'length_limit=' ] ) data_dir = None joint_train = True length_limit = None training_files = [ ] testing_files = [ ] if not exists ( join ( task_dir , 'data' ) ) : mkdir ( join ( task_dir , 'data' ) ) if not exists ( join ( task_dir , 'data' , 'clean' ) ) : mkdir ( join ( task_dir , 'data' , 'clean' ) ) for opt in options : if opt [ 0 ] == '--data_dir' : data_dir = opt [ 1 ] if opt [ 0 ] == '--single_train' : joint_train = False if not exists ( join ( task_dir , 'data' ) ) : mkdir ( join ( task_dir , 'data' ) ) if opt [ 0 ] == '--length_limit' : length_limit = int ( opt [ 1 ] ) if data_dir is None : if exists ( join ( task_dir , 'data' , 'unencoded' ) ) : data_dir = join ( task_dir , 'data' , 'unencoded' ) else : raise ValueError ( "data_dir argument cannot be None" ) all_questions = [ ] for a_file in listdir ( data_dir ) : if a_file . endswith ( ".json" ) : full_path = join ( data_dir , a_file ) with open ( full_path ) as json_file : a_list = json . load ( json_file ) a_list = clean_data ( a_list ) for i , item in enumerate ( a_list ) : write_path = join ( task_dir , 'data' , 'clean' , f"{item[2].replace(' ', '')}_{a_file}" ) with open ( write_path , 'w+' ) as write_file : json . dump ( item , write_file ) if a_file . endswith ( "train.json" ) : training_files . append ( write_path ) else : testing_files . append ( write_path ) all_questions . extend ( a_list ) lexicon_dictionary = create_dictionary ( all_questions ) print ( f"There are {len(lexicon_dictionary)} unique words; " f"{len([entry for entry in lexicon_dictionary if entry.isnumeric()])} of these words are numbers, " f"{len([entry for entry in lexicon_dictionary if entry.isalpha()])} of these words are standard alphabetical" f" words, and " f"{len([entry for entry in lexicon_dictionary if entry in punctuation])} of these words are punctuation " f"marks." ) print ( [ entry for entry in lexicon_dictionary if not entry . isnumeric ( ) and not entry . isalpha ( ) and entry not in punctuation ] ) with open ( join ( task_dir , 'data' , 'dictionary_entries.json' ) , 'w+' ) as write_file : json . dump ( list ( lexicon_dictionary . keys ( ) ) , write_file ) processed_data_dir = join ( task_dir , 'data' , 'encoded' ) train_data_dir = join ( processed_data_dir , 'train' ) test_data_dir = join ( processed_data_dir , 'test' ) if exists ( processed_data_dir ) and isdir ( processed_data_dir ) : rmtree ( processed_data_dir ) mkdir ( processed_data_dir ) mkdir ( train_data_dir ) mkdir ( test_data_dir ) encoded_training_files , training_stories_lengths = encode_data ( training_files , train_data_dir , lexicon_dictionary ) encoded_testing_files , testing_stories_lengths = encode_data ( testing_files , test_data_dir , lexicon_dictionary ) encoded_files = encoded_training_files + encoded_testing_files stories_lengths = np . array ( training_stories_lengths + testing_stories_lengths ) length_limit = np . max ( stories_lengths ) if length_limit is None else length_limit print ( "Total Number of stories: %d" % ( len ( stories_lengths ) ) ) print ( "Number of stories with lengthens > %d: %d (%% %.2f) [discarded]" % ( length_limit , int ( np . sum ( stories_lengths > length_limit ) ) , np . mean ( stories_lengths > length_limit ) * 100.0 ) ) print ( "Number of Remaining Stories: %d" % ( len ( stories_lengths [ stories_lengths <= length_limit ] ) ) ) llprint ( "Saving processed data to disk ... " ) pickle . dump ( lexicon_dictionary , open ( join ( processed_data_dir , 'lexicon-dict.pkl' ) , 'wb' ) ) joint_train_data = [ ] for filename in encoded_files : if filename . endswith ( "test.json" ) : pickle . dump ( encoded_files [ filename ] , open ( join ( test_data_dir , basename ( filename ) + '.pkl' ) , 'wb' ) ) elif filename . endswith ( "train.json" ) : if not joint_train : pickle . dump ( encoded_files [ filename ] , open ( join ( train_data_dir , basename ( filename ) + '.pkl' ) , 'wb' ) ) else : joint_train_data . extend ( encoded_files [ filename ] ) if joint_train : pickle . dump ( joint_train_data , open ( join ( train_data_dir , 'train.pkl' ) , 'wb' ) ) llprint ( "Done!\n" )
Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory
that contains the same files.
Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory that contains the same files .
"""
    Takes json data files in data_dir in the same format as the DREAM dataset and then creates a new directory
    that contains the same files. But in these files, the dialogue, question and answer's words are cleaned. A second
    new directory is also created, this directory stores the cleaned data in its numerical format.
    their numerical equivalent. If single_train is specified, all training and testing dialogues are dumped into the
    same json file. Otherwise, training and testing data is stored separately.
    :return:
    """
None