tasks/DREAM/preprocess.py
def create_dictionary(data):
    """
    Create a dictionary of unique lexicons in the dataset and their mapping to numbers.
    :param data:
    :return:
    """

    lexicons_dict = {}
    id_counter = 0

    llprint("Creating Dictionary ... 0/%d" % (len(data)))

    for index, entry in enumerate(data):
        sentences = entry[0]
        for question_dictionary in entry[1]:
            sentences.append(question_dictionary["question"])
            sentences.append(question_dictionary["answer"])
        for sentence in sentences:
            for word in sentence.split():
                if not word.lower() in lexicons_dict:
                    lexicons_dict[word.lower()] = id_counter
                    id_counter += 1

        llprint("\rCreating Dictionary ... %d/%d" % ((index + 1), len(data)))
    lexicons_dict['+'] = id_counter
    id_counter += 1
    lexicons_dict['\\'] = id_counter
    id_counter += 1
    lexicons_dict['='] = id_counter

    print("\rCreating Dictionary ... Done!")
    return lexicons_dict

    Create a dictionary of unique lexicons in the dataset and their mapping to numbers.
    :param data:
    :return:
    
Create a dictionary of unique lexicons in the dataset and their mapping to numbers.
Create a dictionary of unique lexicons in the dataset and their mapping to numbers .
def create_dictionary(data):
    lexicons_dict = {}
    id_counter = 0
    llprint("Creating Dictionary ... 0/%d" % (len(data)))
    for index, entry in enumerate(data):
        sentences = entry[0]
        for question_dictionary in entry[1]:
            sentences.append(question_dictionary["question"])
            sentences.append(question_dictionary["answer"])
        for sentence in sentences:
            for word in sentence.split():
                if not word.lower() in lexicons_dict:
                    lexicons_dict[word.lower()] = id_counter
                    id_counter += 1
        llprint("\rCreating Dictionary ... %d/%d" % ((index + 1), len(data)))
    lexicons_dict['+'] = id_counter
    id_counter += 1
    lexicons_dict['\\'] = id_counter
    id_counter += 1
    lexicons_dict['='] = id_counter
    print("\rCreating Dictionary ... Done!")
    return lexicons_dict
def create_dictionary ( data ) : lexicons_dict = { } id_counter = 0 llprint ( "Creating Dictionary ... 0/%d" % ( len ( data ) ) ) for index , entry in enumerate ( data ) : sentences = entry [ 0 ] for question_dictionary in entry [ 1 ] : sentences . append ( question_dictionary [ "question" ] ) sentences . append ( question_dictionary [ "answer" ] ) for sentence in sentences : for word in sentence . split ( ) : if not word . lower ( ) in lexicons_dict : lexicons_dict [ word . lower ( ) ] = id_counter id_counter += 1 llprint ( "\rCreating Dictionary ... %d/%d" % ( ( index + 1 ) , len ( data ) ) ) lexicons_dict [ '+' ] = id_counter id_counter += 1 lexicons_dict [ '\\' ] = id_counter id_counter += 1 lexicons_dict [ '=' ] = id_counter print ( "\rCreating Dictionary ... Done!" ) return lexicons_dict
Create a dictionary of unique lexicons in the dataset and their mapping to numbers.
Create a dictionary of unique lexicons in the dataset and their mapping to numbers .
"""
    Create a dictionary of unique lexicons in the dataset and their mapping to numbers.
    :param data:
    :return:
    """
None
data
None