def tokenize(text, normalize_ascii=True):
    """
    Convert a single string into a list of substrings
    split along punctuation and word boundaries. Keep
    whitespace intact by always attaching it to the
    previous token.

    Arguments:
    ----------
        text : str
        normalize_ascii : bool, perform some replacements
            on non-ascii characters to canonicalize the
            string (defaults to True).

    Returns:
    --------
        list<str>, list of substring tokens.
    """
    # 1. If there's no punctuation, return immediately
    if no_punctuation.match(text):
        return [text]
    # 2. let's standardize the input text to ascii (if desired)
    # Note: this will no longer respect input-to-output character positions
    if normalize_ascii:
        # normalize these greco-roman characters to ascii:
        text = text.replace(u"œ", "oe").replace(u"æ", "ae")
        # normalize dashes:
        text = repeated_dash_converter.sub("-", text)
    # 3. let's construct an integer array of the possible split locations:
    split_locations = [UNDECIDED] * len(text)

    regexes = (
        pure_whitespace,
        left_quote_shifter,
        left_quote_converter,
        left_single_quote_converter,
        remaining_quote_converter,
        # regex can't fix this -> regex ca n't fix this
        english_nots,
        # you'll dig this -> you 'll dig this
        english_contractions,
        # the rhino's horns -> the rhino 's horns
        english_specific_appendages,
        # qu'a tu fais au rhino -> qu ' a tu fais au rhino,
        french_appendages
    )
    # 4. Mark end locations for specific regular expressions:
    for regex in regexes:
        mark_regex(regex, text, split_locations)

    begin_end_regexes = (
        multi_single_quote_finder,
        right_single_quote_converter,
        # use dashes as the breakpoint:
        # the rhino--truck -> the rhino -- truck
        simple_dash_finder if normalize_ascii else advanced_dash_finder,
        numerical_expression,
        url_file_finder,
        shifted_ellipses,
        # the #rhino! -> the # rhino ! ;
        # the rino[sic] -> the rino [ sic ]
        shifted_standard_punctuation
    )

    # 5. Mark begin and end locations for other regular expressions:
    for regex in begin_end_regexes:
        mark_begin_end_regex(regex, text, split_locations)

    # 6. Remove splitting on exceptional uses of periods:
    # I'm with Mr. -> I 'm with Mr. , I'm with Mister. -> I 'm with Mister .
    protect_shorthand(text, split_locations)

    if normalize_ascii:
        text = dash_converter.sub("-", text)
    # 7. Return the split string using the integer list:
    return list(split_with_locations(text, split_locations))
def tokenize(text, normalize_ascii=True):
    """
    Convert a single string into a list of substrings
    split along punctuation and word boundaries. Keep
    whitespace intact by always attaching it to the
    previous token.

    Arguments:
    ----------
        text : str
        normalize_ascii : bool, perform some replacements
            on non-ascii characters to canonicalize the
            string (defaults to True).

    Returns:
    --------
        list<str>, list of substring tokens.
    """
    # 1. If there's no punctuation, return immediately
    if no_punctuation.match(text):
        return [text]
    # 2. let's standardize the input text to ascii (if desired)
    # Note: this will no longer respect input-to-output character positions
    if normalize_ascii:
        # normalize these greco-roman characters to ascii:
        text = text.replace(u"œ", "oe").replace(u"æ", "ae")
        # normalize dashes:
        text = repeated_dash_converter.sub("-", text)
    # 3. let's construct an integer array of the possible split locations:
    split_locations = [UNDECIDED] * len(text)

    regexes = (
        pure_whitespace,
        left_quote_shifter,
        left_quote_converter,
        left_single_quote_converter,
        remaining_quote_converter,
        # regex can't fix this -> regex ca n't fix this
        english_nots,
        # you'll dig this -> you 'll dig this
        english_contractions,
        # the rhino's horns -> the rhino 's horns
        english_specific_appendages,
        # qu'a tu fais au rhino -> qu ' a tu fais au rhino,
        french_appendages
    )
    # 4. Mark end locations for specific regular expressions:
    for regex in regexes:
        mark_regex(regex, text, split_locations)

    begin_end_regexes = (
        multi_single_quote_finder,
        right_single_quote_converter,
        # use dashes as the breakpoint:
        # the rhino--truck -> the rhino -- truck
        simple_dash_finder if normalize_ascii else advanced_dash_finder,
        numerical_expression,
        url_file_finder,
        shifted_ellipses,
        # the #rhino! -> the # rhino ! ;
        # the rino[sic] -> the rino [ sic ]
        shifted_standard_punctuation
    )

    # 5. Mark begin and end locations for other regular expressions:
    for regex in begin_end_regexes:
        mark_begin_end_regex(regex, text, split_locations)

    # 6. Remove splitting on exceptional uses of periods:
    # I'm with Mr. -> I 'm with Mr. , I'm with Mister. -> I 'm with Mister .
    protect_shorthand(text, split_locations)

    if normalize_ascii:
        text = dash_converter.sub("-", text)
    # 7. Return the split string using the integer list:
    return list(split_with_locations(text, split_locations))
def tokenize ( text , normalize_ascii = True ) : # 1. If there's no punctuation, return immediately if no_punctuation . match ( text ) : return [ text ] # 2. let's standardize the input text to ascii (if desired) # Note: this will no longer respect input-to-output character positions if normalize_ascii : # normalize these greco-roman characters to ascii: text = text . replace ( u"œ", oe") . r eplace( u "æ",  " e") # normalize dashes: text = repeated_dash_converter . sub ( "-" , text ) # 3. let's construct an integer array of the possible split locations: split_locations = [ UNDECIDED ] * len ( text ) regexes = ( pure_whitespace , left_quote_shifter , left_quote_converter , left_single_quote_converter , remaining_quote_converter , # regex can't fix this -> regex ca n't fix this english_nots , # you'll dig this -> you 'll dig this english_contractions , # the rhino's horns -> the rhino 's horns english_specific_appendages , # qu'a tu fais au rhino -> qu ' a tu fais au rhino, french_appendages ) # 4. Mark end locations for specific regular expressions: for regex in regexes : mark_regex ( regex , text , split_locations ) begin_end_regexes = ( multi_single_quote_finder , right_single_quote_converter , # use dashes as the breakpoint: # the rhino--truck -> the rhino -- truck simple_dash_finder if normalize_ascii else advanced_dash_finder , numerical_expression , url_file_finder , shifted_ellipses , # the #rhino! -> the # rhino ! ; # the rino[sic] -> the rino [ sic ] shifted_standard_punctuation ) # 5. Mark begin and end locations for other regular expressions: for regex in begin_end_regexes : mark_begin_end_regex ( regex , text , split_locations ) # 6. Remove splitting on exceptional uses of periods: # I'm with Mr. -> I 'm with Mr. , I'm with Mister. -> I 'm with Mister . protect_shorthand ( text , split_locations ) if normalize_ascii : text = dash_converter . sub ( "-" , text ) # 7. Return the split string using the integer list: return list ( split_with_locations ( text , split_locations ) )
Convert a single string into a list of substrings
    split along punctuation and word boundaries. Keep
    whitespace intact by always attaching it to the
    previous token.

    Arguments:
    ----------
        text : str
        normalize_ascii : bool, perform some replacements
            on non-ascii characters to canonicalize the
            string (defaults to True).

    Returns:
    --------
        list<str>, list of substring tokens.
Convert a single string into a list of substrings split along punctuation and word boundaries . Keep whitespace intact by always attaching it to the previous token .