text_extensions_for_pandas/spanner/extract.py
def extract_regex_tok(
    tokens: Union[SpanArray, pd.Series],
    compiled_regex: regex.Regex,
    min_len=1,
    max_len=1,
    output_col_name: str = "match",
):
    """
    Identify all (possibly overlapping) matches of a regular expression
    that start and end on token boundaries.

    :param tokens: ``SpanArray`` of token information, optionally wrapped in a
     `pd.Series`.
    :param compiled_regex: Regular expression to evaluate.
    :param min_len: Minimum match length in tokens
    :param max_len: Maximum match length (inclusive) in tokens
    :param output_col_name: (optional) name of column of matching spans in the
     returned DataFrame

    :returns: A single-column DataFrame containing a span for each match of the
     regex.
    """
    tokens = SpanArray.make_array(tokens)

    num_tokens = len(tokens)
    matches_regex_f = np.vectorize(lambda s: compiled_regex.fullmatch(s) is not None)

    # The built-in regex functionality of Pandas/Python does not have
    # an optimized single-pass RegexTok, so generate all the places
    # where there might be a match and run them through regex.fullmatch().
    # Note that this approach is asymptotically inefficient if max_len is large.
    # TODO: Performance tuning for both small and large max_len
    matches_list = []
    for cur_len in range(min_len, max_len + 1):
        window_begin_toks = np.arange(0, num_tokens - cur_len + 1)
        window_end_toks = window_begin_toks + cur_len

        window_tok_spans = TokenSpanArray(tokens, window_begin_toks, window_end_toks)
        matches_list.append(
            pd.Series(window_tok_spans[matches_regex_f(window_tok_spans.covered_text)])
        )
    return pd.DataFrame({output_col_name: pd.concat(matches_list)})

    Identify all (possibly overlapping) matches of a regular expression
    that start and end on token boundaries.

    :param tokens: ``SpanArray`` of token information, optionally wrapped in a
     `pd.Series`.
    :param compiled_regex: Regular expression to evaluate.
    :param min_len: Minimum match length in tokens
    :param max_len: Maximum match length (inclusive) in tokens
    :param output_col_name: (optional) name of column of matching spans in the
     returned DataFrame

    :returns: A single-column DataFrame containing a span for each match of the
     regex.
    
Identify all (possibly overlapping) matches of a regular expression
that start and end on token boundaries.
Identify all ( possibly overlapping ) matches of a regular expression that start and end on token boundaries .
def extract_regex_tok(
    tokens: Union[SpanArray, pd.Series],
    compiled_regex: regex.Regex,
    min_len=1,
    max_len=1,
    output_col_name: str = "match",
):
    tokens = SpanArray.make_array(tokens)
    num_tokens = len(tokens)
    matches_regex_f = np.vectorize(lambda s: compiled_regex.fullmatch(s) is not None)
    matches_list = []
    for cur_len in range(min_len, max_len + 1):
        window_begin_toks = np.arange(0, num_tokens - cur_len + 1)
        window_end_toks = window_begin_toks + cur_len
        window_tok_spans = TokenSpanArray(tokens, window_begin_toks, window_end_toks)
        matches_list.append(
            pd.Series(window_tok_spans[matches_regex_f(window_tok_spans.covered_text)])
        )
    return pd.DataFrame({output_col_name: pd.concat(matches_list)})
def extract_regex_tok ( tokens : Union [ SpanArray , pd . Series ] , compiled_regex : regex . Regex , min_len = 1 , max_len = 1 , output_col_name : str = "match" , ) : tokens = SpanArray . make_array ( tokens ) num_tokens = len ( tokens ) matches_regex_f = np . vectorize ( lambda s : compiled_regex . fullmatch ( s ) is not None ) matches_list = [ ] for cur_len in range ( min_len , max_len + 1 ) : window_begin_toks = np . arange ( 0 , num_tokens - cur_len + 1 ) window_end_toks = window_begin_toks + cur_len window_tok_spans = TokenSpanArray ( tokens , window_begin_toks , window_end_toks ) matches_list . append ( pd . Series ( window_tok_spans [ matches_regex_f ( window_tok_spans . covered_text ) ] ) ) return pd . DataFrame ( { output_col_name : pd . concat ( matches_list ) } )
Identify all (possibly overlapping) matches of a regular expression
that start and end on token boundaries.
Identify all ( possibly overlapping ) matches of a regular expression that start and end on token boundaries .
"""
    Identify all (possibly overlapping) matches of a regular expression
    that start and end on token boundaries.

    :param tokens: ``SpanArray`` of token information, optionally wrapped in a
     `pd.Series`.
    :param compiled_regex: Regular expression to evaluate.
    :param min_len: Minimum match length in tokens
    :param max_len: Maximum match length (inclusive) in tokens
    :param output_col_name: (optional) name of column of matching spans in the
     returned DataFrame

    :returns: A single-column DataFrame containing a span for each match of the
     regex.
    """ # The built-in regex functionality of Pandas/Python does not have # an optimized single-pass RegexTok, so generate all the places # where there might be a match and run them through regex.fullmatch(). # Note that this approach is asymptotically inefficient if max_len is large. # TODO: Performance tuning for both small and large max_len
A single-column DataFrame containing a span for each match of the
regex.
A single - column DataFrame containing a span for each match of the regex .
tokens
Union[SpanArray, pd.Series]
``SpanArray`` of token information, optionally wrapped in a
`pd.Series`.
` ` SpanArray ` ` of token information optionally wrapped in a ` pd . Series ` .
compiled_regex
regex.Regex
Regular expression to evaluate.
Regular expression to evaluate .
min_len
Minimum match length in tokens
Minimum match length in tokens
max_len
Maximum match length (inclusive) in tokens
Maximum match length ( inclusive ) in tokens
output_col_name
str
(optional) name of column of matching spans in the
returned DataFrame
( optional ) name of column of matching spans in the returned DataFrame