tasks/DREAM/preprocess.py
def clean_sentences(sentence_list):
    """
    Cleans sentence_list by: indicating title words by placing a separate word "\^{}" in front of the capitalized word,
    indicating all-caps word by placing a separate word "\^{}\^{}" in front of the all-caps word, making all words
    lower case, fixing spelling errors, separating units from numbers, giving all onomatopoeia words the same spelling,
    separating two words that were missing a space, changing the ending of possessor nouns ending in s, expanding
    abbreviations, making all number into their own words, changing numbers that were spelled out into digits, making
    all digits into their own separate words and breaking hyphenated words apart.
    :param sentence_list: A list of sentences.
    :return: The modified sentence list.
    """
    new_sentence_list = []
    for sentence in sentence_list:
        if sentence in sentence_dict.keys():
            sentence_list[sentence_list.index(sentence)] = sentence_dict[sentence]

    for index, sentence in enumerate(sentence_list):
        capitalized = set()
        abbreviations = set()
        for word in sentence.split():
            if word.istitle():
                capitalized.add(word)
            elif word.isupper():
                abbreviations.add(word)

        new_sentence = sentence.split()
        for i in range(len(new_sentence)):
            if new_sentence[i] in capitalized:
                new_sentence[i] = f" ^ {new_sentence[i]}"
            if new_sentence[i] in abbreviations:
                new_sentence[i] = f" ^^ {new_sentence[i]}"
        new_sentence = " ".join(new_sentence)
        new_sentence = new_sentence.lower()

        for symbol in symbols.keys():
            if symbol in new_sentence:
                new_sentence = new_sentence.replace(symbol, symbols[symbol])

        words = new_sentence.split()
        for word in words:
            for pattern in spelling_regex.keys():
                match = re.fullmatch(pattern, word)
                if match:
                    new_sentence = new_sentence.split()
                    for i in range(len(new_sentence)):
                        if new_sentence[i] == word:
                            new_sentence[i] = spelling_regex[pattern]
                    new_sentence = " ".join(new_sentence)
            for pattern in units.keys():
                match = re.fullmatch(pattern, word)
                if match:
                    if pattern == r"\d{2,}s-\d{2,}s":
                        new_sentence = new_sentence.split()
                        for i in range(len(new_sentence)):
                            if new_sentence[i] == word:
                                new_sentence[i] = word[:word.find('s')] + " s - " + word[word.find("-") + 1:-1] + " s"
                        new_sentence = " ".join(new_sentence)
                    else:
                        new_sentence = new_sentence.split()
                        for i in range(len(new_sentence)):
                            if new_sentence[i] == word:
                                new_sentence[i] = word[:word.find(units[pattern])] + ' ' + units[pattern]
                        new_sentence = " ".join(new_sentence)
            for pattern in onomatopoeia:
                match = re.fullmatch(pattern, word)
                if match:
                    new_sentence = new_sentence.split()
                    for i in range(len(new_sentence)):
                        if new_sentence[i] == word:
                            new_sentence[i] = onomatopoeia[pattern]
                    new_sentence = " ".join(new_sentence)

        word_array = clean_word_array(new_sentence.split())
        new_sentence = " ".join(word_array)

        new_sentence = new_sentence.replace('-', ' - ')

        serial_names = set()
        for word in new_sentence.split():
            if bool(re.match('^(?=.*[0-9])(?=.*[a-z])', word)):
                serial_names.add(word)
        new_sentence = new_sentence.split()
        for i in range(len(new_sentence)):
            if new_sentence[i] in serial_names:
                new_sentence[i] = ' '.join([f"^ {character}" if character.isalpha() else character for character in
                                            new_sentence[i]])
        new_sentence = " ".join(new_sentence)

        for word in new_sentence.split():
            if word == '×':
                print(new_sentence)

        new_sentence_list.append(new_sentence)

    return new_sentence_list

    Cleans sentence_list by: indicating title words by placing a separate word "\^{}" in front of the capitalized word,
    indicating all-caps word by placing a separate word "\^{}\^{}" in front of the all-caps word, making all words
    lower case, fixing spelling errors, separating units from numbers, giving all onomatopoeia words the same spelling,
    separating two words that were missing a space, changing the ending of possessor nouns ending in s, expanding
    abbreviations, making all number into their own words, changing numbers that were spelled out into digits, making
    all digits into their own separate words and breaking hyphenated words apart.
    :param sentence_list: A list of sentences.
    :return: The modified sentence list.
    
Cleans sentence_list by: indicating title words by placing a separate word "\^{}" in front of the capitalized word,
indicating all-caps word by placing a separate word "\^{}\^{}" in front of the all-caps word, making all words
lower case, fixing spelling errors, separating units from numbers, giving all onomatopoeia words the same spelling,
separating two words that were missing a space, changing the ending of possessor nouns ending in s, expanding
abbreviations, making all number into their own words, changing numbers that were spelled out into digits, making
all digits into their own separate words and breaking hyphenated words apart.
Cleans sentence_list by : indicating title words by placing a separate word " \ ^ {} " in front of the capitalized word indicating all - caps word by placing a separate word " \ ^ {} \ ^ {} " in front of the all - caps word making all words lower case fixing spelling errors separating units from numbers giving all onomatopoeia words the same spelling separating two words that were missing a space changing the ending of possessor nouns ending in s expanding abbreviations making all number into their own words changing numbers that were spelled out into digits making all digits into their own separate words and breaking hyphenated words apart .
def clean_sentences(sentence_list):
    new_sentence_list = []
    for sentence in sentence_list:
        if sentence in sentence_dict.keys():
            sentence_list[sentence_list.index(sentence)] = sentence_dict[sentence]
    for index, sentence in enumerate(sentence_list):
        capitalized = set()
        abbreviations = set()
        for word in sentence.split():
            if word.istitle():
                capitalized.add(word)
            elif word.isupper():
                abbreviations.add(word)
        new_sentence = sentence.split()
        for i in range(len(new_sentence)):
            if new_sentence[i] in capitalized:
                new_sentence[i] = f" ^ {new_sentence[i]}"
            if new_sentence[i] in abbreviations:
                new_sentence[i] = f" ^^ {new_sentence[i]}"
        new_sentence = " ".join(new_sentence)
        new_sentence = new_sentence.lower()
        for symbol in symbols.keys():
            if symbol in new_sentence:
                new_sentence = new_sentence.replace(symbol, symbols[symbol])
        words = new_sentence.split()
        for word in words:
            for pattern in spelling_regex.keys():
                match = re.fullmatch(pattern, word)
                if match:
                    new_sentence = new_sentence.split()
                    for i in range(len(new_sentence)):
                        if new_sentence[i] == word:
                            new_sentence[i] = spelling_regex[pattern]
                    new_sentence = " ".join(new_sentence)
            for pattern in units.keys():
                match = re.fullmatch(pattern, word)
                if match:
                    if pattern == r"\d{2,}s-\d{2,}s":
                        new_sentence = new_sentence.split()
                        for i in range(len(new_sentence)):
                            if new_sentence[i] == word:
                                new_sentence[i] = word[:word.find('s')] + " s - " + word[word.find("-") + 1:-1] + " s"
                        new_sentence = " ".join(new_sentence)
                    else:
                        new_sentence = new_sentence.split()
                        for i in range(len(new_sentence)):
                            if new_sentence[i] == word:
                                new_sentence[i] = word[:word.find(units[pattern])] + ' ' + units[pattern]
                        new_sentence = " ".join(new_sentence)
            for pattern in onomatopoeia:
                match = re.fullmatch(pattern, word)
                if match:
                    new_sentence = new_sentence.split()
                    for i in range(len(new_sentence)):
                        if new_sentence[i] == word:
                            new_sentence[i] = onomatopoeia[pattern]
                    new_sentence = " ".join(new_sentence)
        word_array = clean_word_array(new_sentence.split())
        new_sentence = " ".join(word_array)
        new_sentence = new_sentence.replace('-', ' - ')
        serial_names = set()
        for word in new_sentence.split():
            if bool(re.match('^(?=.*[0-9])(?=.*[a-z])', word)):
                serial_names.add(word)
        new_sentence = new_sentence.split()
        for i in range(len(new_sentence)):
            if new_sentence[i] in serial_names:
                new_sentence[i] = ' '.join([f"^ {character}" if character.isalpha() else character for character in
                                            new_sentence[i]])
        new_sentence = " ".join(new_sentence)
        for word in new_sentence.split():
            if word == '×':
                print(new_sentence)
        new_sentence_list.append(new_sentence)
    return new_sentence_list
def clean_sentences ( sentence_list ) : new_sentence_list = [ ] for sentence in sentence_list : if sentence in sentence_dict . keys ( ) : sentence_list [ sentence_list . index ( sentence ) ] = sentence_dict [ sentence ] for index , sentence in enumerate ( sentence_list ) : capitalized = set ( ) abbreviations = set ( ) for word in sentence . split ( ) : if word . istitle ( ) : capitalized . add ( word ) elif word . isupper ( ) : abbreviations . add ( word ) new_sentence = sentence . split ( ) for i in range ( len ( new_sentence ) ) : if new_sentence [ i ] in capitalized : new_sentence [ i ] = f" ^ {new_sentence[i]}" if new_sentence [ i ] in abbreviations : new_sentence [ i ] = f" ^^ {new_sentence[i]}" new_sentence = " " . join ( new_sentence ) new_sentence = new_sentence . lower ( ) for symbol in symbols . keys ( ) : if symbol in new_sentence : new_sentence = new_sentence . replace ( symbol , symbols [ symbol ] ) words = new_sentence . split ( ) for word in words : for pattern in spelling_regex . keys ( ) : match = re . fullmatch ( pattern , word ) if match : new_sentence = new_sentence . split ( ) for i in range ( len ( new_sentence ) ) : if new_sentence [ i ] == word : new_sentence [ i ] = spelling_regex [ pattern ] new_sentence = " " . join ( new_sentence ) for pattern in units . keys ( ) : match = re . fullmatch ( pattern , word ) if match : if pattern == r"\d{2,}s-\d{2,}s" : new_sentence = new_sentence . split ( ) for i in range ( len ( new_sentence ) ) : if new_sentence [ i ] == word : new_sentence [ i ] = word [ : word . find ( 's' ) ] + " s - " + word [ word . find ( "-" ) + 1 : - 1 ] + " s" new_sentence = " " . join ( new_sentence ) else : new_sentence = new_sentence . split ( ) for i in range ( len ( new_sentence ) ) : if new_sentence [ i ] == word : new_sentence [ i ] = word [ : word . find ( units [ pattern ] ) ] + ' ' + units [ pattern ] new_sentence = " " . join ( new_sentence ) for pattern in onomatopoeia : match = re . fullmatch ( pattern , word ) if match : new_sentence = new_sentence . split ( ) for i in range ( len ( new_sentence ) ) : if new_sentence [ i ] == word : new_sentence [ i ] = onomatopoeia [ pattern ] new_sentence = " " . join ( new_sentence ) word_array = clean_word_array ( new_sentence . split ( ) ) new_sentence = " " . join ( word_array ) new_sentence = new_sentence . replace ( '-' , ' - ' ) serial_names = set ( ) for word in new_sentence . split ( ) : if bool ( re . match ( '^(?=.*[0-9])(?=.*[a-z])' , word ) ) : serial_names . add ( word ) new_sentence = new_sentence . split ( ) for i in range ( len ( new_sentence ) ) : if new_sentence [ i ] in serial_names : new_sentence [ i ] = ' ' . join ( [ f"^ {character}" if character . isalpha ( ) else character for character in new_sentence [ i ] ] ) new_sentence = " " . join ( new_sentence ) for word in new_sentence . split ( ) : if word == '×': print ( new_sentence ) new_sentence_list . append ( new_sentence ) return new_sentence_list
Cleans sentence_list by: indicating title words by placing a separate word "\^{}" in front of the capitalized word,
indicating all-caps word by placing a separate word "\^{}\^{}" in front of the all-caps word, making all words
lower case, fixing spelling errors, separating units from numbers, giving all onomatopoeia words the same spelling,
separating two words that were missing a space, changing the ending of possessor nouns ending in s, expanding
abbreviations, making all number into their own words, changing numbers that were spelled out into digits, making
all digits into their own separate words and breaking hyphenated words apart.
Cleans sentence_list by : indicating title words by placing a separate word " \ ^ {} " in front of the capitalized word indicating all - caps word by placing a separate word " \ ^ {} \ ^ {} " in front of the all - caps word making all words lower case fixing spelling errors separating units from numbers giving all onomatopoeia words the same spelling separating two words that were missing a space changing the ending of possessor nouns ending in s expanding abbreviations making all number into their own words changing numbers that were spelled out into digits making all digits into their own separate words and breaking hyphenated words apart .
"""
    Cleans sentence_list by: indicating title words by placing a separate word "\^{}" in front of the capitalized word,
    indicating all-caps word by placing a separate word "\^{}\^{}" in front of the all-caps word, making all words
    lower case, fixing spelling errors, separating units from numbers, giving all onomatopoeia words the same spelling,
    separating two words that were missing a space, changing the ending of possessor nouns ending in s, expanding
    abbreviations, making all number into their own words, changing numbers that were spelled out into digits, making
    all digits into their own separate words and breaking hyphenated words apart.
    :param sentence_list: A list of sentences.
    :return: The modified sentence list.
    """
The modified sentence list.
The modified sentence list .
sentence_list
A list of sentences.
A list of sentences .