ciseau/sentence_tokenizer.py
def detect_sentence_boundaries(tokens):
    """
    Subdivide an input list of strings (tokens)
    into multiple lists according to detected
    sentence boundaries.

    ```
    detect_sentence_boundaries(
        ["Cat ", "sat ", "mat", ". ", "Cat ", "'s ", "named ", "Cool", "."]
    )
    #=> [
        ["Cat ", "sat ", "mat", ". "],
        ["Cat ", "'s ", "named ", "Cool", "."]
    ]
    ```

    Arguments:
    ----------

        tokens : list<str>

    Returns:
    --------
        list<list<str>> : original list subdivided into multiple
            lists according to (detected) sentence boundaries.
    """
    tokenized = group_quoted_tokens(tokens)
    words = []
    sentences = []
    for i in range(len(tokenized)):
        # this is a parenthetical:
        end_sentence = False
        if isinstance(tokenized[i], list):
            if len(words) == 0:
                # end if a sentence finishes inside quoted section,
                # and no sentence was begun beforehand
                if is_end_symbol(tokenized[i][-2].rstrip()):
                    end_sentence = True
            else:
                # end if a sentence finishes inside quote marks
                if (tokenized[i][0][0] == '"' and
                    is_end_symbol(tokenized[i][-2].rstrip()) and
                    not tokenized[i][1][0].isupper()):
                    end_sentence = True
            words.extend(tokenized[i])
        else:
            stripped_tokenized = tokenized[i].rstrip()
            if is_end_symbol(stripped_tokenized):
                words.append(tokenized[i])
                not_last_word = i + 1 != len(tokenized)
                next_word_lowercase = (
                    not_last_word and
                    tokenized[i+1][0].islower()
                )
                next_word_continue_punct = (
                    not_last_word and
                    tokenized[i+1][0] in CONTINUE_PUNCT_SYMBOLS
                )
                end_sentence = not (
                    not_last_word and
                    (
                        next_word_lowercase or
                        next_word_continue_punct
                    )
                )
            else:
                words.append(tokenized[i])
        if end_sentence:
            sentences.append(words)
            words = []

    # add final sentence, if it wasn't added yet.
    if len(words) > 0:
        sentences.append(words)

    # If the final word ends in a period:
    if len(sentences) > 0 and sentences[-1][-1]:
        alpha_word_piece = word_with_alpha_and_period.match(sentences[-1][-1])
        if alpha_word_piece:
            sentences[-1][-1] = alpha_word_piece.group(1)
            sentences[-1].append(alpha_word_piece.group(2))
    return sentences
def detect_sentence_boundaries(tokens):
    """
    Subdivide an input list of strings (tokens)
    into multiple lists according to detected
    sentence boundaries.

    ```
    detect_sentence_boundaries(
        ["Cat ", "sat ", "mat", ". ", "Cat ", "'s ", "named ", "Cool", "."]
    )
    #=> [
        ["Cat ", "sat ", "mat", ". "],
        ["Cat ", "'s ", "named ", "Cool", "."]
    ]
    ```

    Arguments:
    ----------

        tokens : list<str>

    Returns:
    --------
        list<list<str>> : original list subdivided into multiple
            lists according to (detected) sentence boundaries.
    """
    tokenized = group_quoted_tokens(tokens)
    words = []
    sentences = []
    for i in range(len(tokenized)):
        # this is a parenthetical:
        end_sentence = False
        if isinstance(tokenized[i], list):
            if len(words) == 0:
                # end if a sentence finishes inside quoted section,
                # and no sentence was begun beforehand
                if is_end_symbol(tokenized[i][-2].rstrip()):
                    end_sentence = True
            else:
                # end if a sentence finishes inside quote marks
                if (tokenized[i][0][0] == '"' and
                    is_end_symbol(tokenized[i][-2].rstrip()) and
                    not tokenized[i][1][0].isupper()):
                    end_sentence = True
            words.extend(tokenized[i])
        else:
            stripped_tokenized = tokenized[i].rstrip()
            if is_end_symbol(stripped_tokenized):
                words.append(tokenized[i])
                not_last_word = i + 1 != len(tokenized)
                next_word_lowercase = (
                    not_last_word and
                    tokenized[i+1][0].islower()
                )
                next_word_continue_punct = (
                    not_last_word and
                    tokenized[i+1][0] in CONTINUE_PUNCT_SYMBOLS
                )
                end_sentence = not (
                    not_last_word and
                    (
                        next_word_lowercase or
                        next_word_continue_punct
                    )
                )
            else:
                words.append(tokenized[i])
        if end_sentence:
            sentences.append(words)
            words = []

    # add final sentence, if it wasn't added yet.
    if len(words) > 0:
        sentences.append(words)

    # If the final word ends in a period:
    if len(sentences) > 0 and sentences[-1][-1]:
        alpha_word_piece = word_with_alpha_and_period.match(sentences[-1][-1])
        if alpha_word_piece:
            sentences[-1][-1] = alpha_word_piece.group(1)
            sentences[-1].append(alpha_word_piece.group(2))
    return sentences
def detect_sentence_boundaries ( tokens ) : tokenized = group_quoted_tokens ( tokens ) words = [ ] sentences = [ ] for i in range ( len ( tokenized ) ) : # this is a parenthetical: end_sentence = False if isinstance ( tokenized [ i ] , list ) : if len ( words ) == 0 : # end if a sentence finishes inside quoted section, # and no sentence was begun beforehand if is_end_symbol ( tokenized [ i ] [ - 2 ] . rstrip ( ) ) : end_sentence = True else : # end if a sentence finishes inside quote marks if ( tokenized [ i ] [ 0 ] [ 0 ] == '"' and is_end_symbol ( tokenized [ i ] [ - 2 ] . rstrip ( ) ) and not tokenized [ i ] [ 1 ] [ 0 ] . isupper ( ) ) : end_sentence = True words . extend ( tokenized [ i ] ) else : stripped_tokenized = tokenized [ i ] . rstrip ( ) if is_end_symbol ( stripped_tokenized ) : words . append ( tokenized [ i ] ) not_last_word = i + 1 != len ( tokenized ) next_word_lowercase = ( not_last_word and tokenized [ i + 1 ] [ 0 ] . islower ( ) ) next_word_continue_punct = ( not_last_word and tokenized [ i + 1 ] [ 0 ] in CONTINUE_PUNCT_SYMBOLS ) end_sentence = not ( not_last_word and ( next_word_lowercase or next_word_continue_punct ) ) else : words . append ( tokenized [ i ] ) if end_sentence : sentences . append ( words ) words = [ ] # add final sentence, if it wasn't added yet. if len ( words ) > 0 : sentences . append ( words ) # If the final word ends in a period: if len ( sentences ) > 0 and sentences [ - 1 ] [ - 1 ] : alpha_word_piece = word_with_alpha_and_period . match ( sentences [ - 1 ] [ - 1 ] ) if alpha_word_piece : sentences [ - 1 ] [ - 1 ] = alpha_word_piece . group ( 1 ) sentences [ - 1 ] . append ( alpha_word_piece . group ( 2 ) ) return sentences
Subdivide an input list of strings (tokens)
    into multiple lists according to detected
    sentence boundaries.

    ```
    detect_sentence_boundaries(
        ["Cat ", "sat ", "mat", ". ", "Cat ", "'s ", "named ", "Cool", "."]
    )
    #=> [
        ["Cat ", "sat ", "mat", ". "],
        ["Cat ", "'s ", "named ", "Cool", "."]
    ]
    ```

    Arguments:
    ----------

        tokens : list<str>

    Returns:
    --------
        list<list<str>> : original list subdivided into multiple
            lists according to (detected) sentence boundaries.
Subdivide an input list of strings ( tokens ) into multiple lists according to detected sentence boundaries .