tasks/DREAM/preprocess.py
def encode_data(files_list, encoded_dir, lexicon_dictionary):
    """
    Convert open files in files_list, convert their words into numerical equivalents
    as defined in lexicon_dictionary, and then store the encoded information in a
    file of the same name but which is located in encoded_dir.
    :param files_list: The list of files that should have their information converted
    :param encoded_dir: The directory to store the new, encoded files in.
    :param lexicon_dictionary: The dictionary mapping words to numbers (here, numbers
            represent the high bit index for a one-hot vector encoding).
    :return: the list of paths containing the encoded information.
    """
    story_inputs = []
    stories_lengths = []

    llprint("Encoding Data ... 0/%d" % (len(files_list)))
    for index, file_path in enumerate(files_list):
        write_path = join(encoded_dir, basename(file_path)[:basename(file_path).rfind('.json')])
        with open(file_path) as data:
            entry = json.load(data)
            for j in range(len(entry)):
                if j == 0:
                    for line in entry[j]:
                        encoded_line = encode_sentences(line, lexicon_dictionary)
                        if len(story_inputs) == 0:
                            story_inputs = encoded_line
                        else:
                            story_inputs.append(lexicon_dictionary["+"])
                            story_inputs.extend(encoded_line)
                    story_inputs.append(lexicon_dictionary["\\"])
                elif j == 1:
                    for i, question_dictionary in enumerate(entry[j]):
                        question = encode_sentences(question_dictionary["question"], lexicon_dictionary)

                        answer = [lexicon_dictionary["="]]
                        answer.extend(encode_sentences(question_dictionary["answer"], lexicon_dictionary))
                        full_list = story_inputs + question + answer
                        with open(write_path + f"{i}.json", 'w+') as write_file:
                            json.dump(full_list, write_file)

                        stories_lengths.append(len(full_list))
        story_inputs = []
        files_list[index] = write_path

        llprint("\rEncoding Data ... %d/%d" % (index + 1, len(files_list)))

    print("\rEncoding Data ... Done!")
    return files_list, stories_lengths

    Convert open files in files_list, convert their words into numerical equivalents
    as defined in lexicon_dictionary, and then store the encoded information in a
    file of the same name but which is located in encoded_dir.
    :param files_list: The list of files that should have their information converted
    :param encoded_dir: The directory to store the new, encoded files in.
    :param lexicon_dictionary: The dictionary mapping words to numbers (here, numbers
            represent the high bit index for a one-hot vector encoding).
    :return: the list of paths containing the encoded information.
    
Convert open files in files_list, convert their words into numerical equivalents
as defined in lexicon_dictionary, and then store the encoded information in a
file of the same name but which is located in encoded_dir.
Convert open files in files_list convert their words into numerical equivalents as defined in lexicon_dictionary and then store the encoded information in a file of the same name but which is located in encoded_dir .
def encode_data(files_list, encoded_dir, lexicon_dictionary):
    story_inputs = []
    stories_lengths = []
    llprint("Encoding Data ... 0/%d" % (len(files_list)))
    for index, file_path in enumerate(files_list):
        write_path = join(encoded_dir, basename(file_path)[:basename(file_path).rfind('.json')])
        with open(file_path) as data:
            entry = json.load(data)
            for j in range(len(entry)):
                if j == 0:
                    for line in entry[j]:
                        encoded_line = encode_sentences(line, lexicon_dictionary)
                        if len(story_inputs) == 0:
                            story_inputs = encoded_line
                        else:
                            story_inputs.append(lexicon_dictionary["+"])
                            story_inputs.extend(encoded_line)
                    story_inputs.append(lexicon_dictionary["\\"])
                elif j == 1:
                    for i, question_dictionary in enumerate(entry[j]):
                        question = encode_sentences(question_dictionary["question"], lexicon_dictionary)
                        answer = [lexicon_dictionary["="]]
                        answer.extend(encode_sentences(question_dictionary["answer"], lexicon_dictionary))
                        full_list = story_inputs + question + answer
                        with open(write_path + f"{i}.json", 'w+') as write_file:
                            json.dump(full_list, write_file)
                        stories_lengths.append(len(full_list))
        story_inputs = []
        files_list[index] = write_path
        llprint("\rEncoding Data ... %d/%d" % (index + 1, len(files_list)))
    print("\rEncoding Data ... Done!")
    return files_list, stories_lengths
def encode_data ( files_list , encoded_dir , lexicon_dictionary ) : story_inputs = [ ] stories_lengths = [ ] llprint ( "Encoding Data ... 0/%d" % ( len ( files_list ) ) ) for index , file_path in enumerate ( files_list ) : write_path = join ( encoded_dir , basename ( file_path ) [ : basename ( file_path ) . rfind ( '.json' ) ] ) with open ( file_path ) as data : entry = json . load ( data ) for j in range ( len ( entry ) ) : if j == 0 : for line in entry [ j ] : encoded_line = encode_sentences ( line , lexicon_dictionary ) if len ( story_inputs ) == 0 : story_inputs = encoded_line else : story_inputs . append ( lexicon_dictionary [ "+" ] ) story_inputs . extend ( encoded_line ) story_inputs . append ( lexicon_dictionary [ "\\" ] ) elif j == 1 : for i , question_dictionary in enumerate ( entry [ j ] ) : question = encode_sentences ( question_dictionary [ "question" ] , lexicon_dictionary ) answer = [ lexicon_dictionary [ "=" ] ] answer . extend ( encode_sentences ( question_dictionary [ "answer" ] , lexicon_dictionary ) ) full_list = story_inputs + question + answer with open ( write_path + f"{i}.json" , 'w+' ) as write_file : json . dump ( full_list , write_file ) stories_lengths . append ( len ( full_list ) ) story_inputs = [ ] files_list [ index ] = write_path llprint ( "\rEncoding Data ... %d/%d" % ( index + 1 , len ( files_list ) ) ) print ( "\rEncoding Data ... Done!" ) return files_list , stories_lengths
Convert open files in files_list, convert their words into numerical equivalents
as defined in lexicon_dictionary, and then store the encoded information in a
file of the same name but which is located in encoded_dir.
Convert open files in files_list convert their words into numerical equivalents as defined in lexicon_dictionary and then store the encoded information in a file of the same name but which is located in encoded_dir .
"""
    Convert open files in files_list, convert their words into numerical equivalents
    as defined in lexicon_dictionary, and then store the encoded information in a
    file of the same name but which is located in encoded_dir.
    :param files_list: The list of files that should have their information converted
    :param encoded_dir: The directory to store the new, encoded files in.
    :param lexicon_dictionary: The dictionary mapping words to numbers (here, numbers
            represent the high bit index for a one-hot vector encoding).
    :return: the list of paths containing the encoded information.
    """
the list of paths containing the encoded information.
the list of paths containing the encoded information .
files_list
The list of files that should have their information converted
The list of files that should have their information converted
encoded_dir
The directory to store the new, encoded files in.
The directory to store the new encoded files in .
lexicon_dictionary
The dictionary mapping words to numbers (here, numbers
represent the high bit index for a one-hot vector encoding).
The dictionary mapping words to numbers ( here numbers represent the high bit index for a one - hot vector encoding ) .