text_extensions_for_pandas/spanner/extract.py
def extract_dict(
    tokens: Union[SpanArray, pd.Series],
    dictionary: pd.DataFrame,
    output_col_name: str = "match",
):
    """
    Identify all matches of a dictionary on a sequence of tokens.

    :param tokens: `SpanArray` of token information, optionally wrapped in a
     `pd.Series`.
    :param dictionary: The dictionary to match, encoded as a `pd.DataFrame` in
     the format returned by `load_dict()`
    :param output_col_name: (optional) name of column of matching spans in the
     returned DataFrame

    :return: a single-column DataFrame of token ID spans of dictionary matches
    """
    # Box tokens into a pd.Series if not already boxed.
    if isinstance(tokens, SpanArray):
        tokens = pd.Series(tokens)

    # Wrap the important parts of the tokens series in a temporary dataframe.
    # noinspection PyUnresolvedReferences
    toks_tmp = pd.DataFrame(
        {
            "token_id": tokens.index,
            "normalized_text": tokens.array.normalized_covered_text,
        }
    )

    # Start by matching the first token.
    matches = pd.merge(
        dictionary, toks_tmp, left_on="toks_0", right_on="normalized_text"
    )
    matches.rename(columns={"token_id": "begin_token_id"}, inplace=True)
    matches_col_names = list(matches.columns)  # We'll need this later

    # Check against remaining elements of matching dictionary entries and
    # accumulate the full set of matches as a list of IntervalIndexes
    begins_list = []
    ends_list = []
    max_entry_len = len(dictionary.columns)
    for match_len in range(1, max_entry_len):
        # print("Match len: {}".format(match_len))
        # Find matches of length match_len. Dictionary entries of this length
        # will have None in the column "toks_<match_len>".
        match_locs = pd.isna(matches["toks_{}".format(match_len)])
        # print("Completed matches:\n{}".format(matches[match_locs]))
        match_begins = matches[match_locs]["begin_token_id"].to_numpy()
        match_ends = match_begins + match_len
        begins_list.append(match_begins)
        ends_list.append(match_ends)

        # For the remaining partial matches against longer dictionary entries,
        # check the next token by merging with the tokens dataframe.
        potential_matches = matches[~match_locs].copy()
        # print("Raw potential matches:\n{}".format(potential_matches))
        potential_matches.drop("normalized_text", axis=1, inplace=True)
        potential_matches["next_token_id"] = (
            potential_matches["begin_token_id"] + match_len
        )
        potential_matches = pd.merge(
            potential_matches, toks_tmp, left_on="next_token_id", right_on="token_id"
        )
        # print("Filtered potential matches:\n{}".format(potential_matches))
        potential_matches = potential_matches[
            potential_matches["normalized_text"]
            == potential_matches["toks_{}".format(match_len)]
        ]
        # The result of the join has some extra columns that we don't need.
        matches = potential_matches[matches_col_names]
    # Gather together all the sets of matches and wrap in a dataframe.
    begins = np.concatenate(begins_list)
    ends = np.concatenate(ends_list)
    result = pd.DataFrame(
        {output_col_name: TokenSpanArray(tokens.values, begins, ends)}
    )
    # Results are sorted by number of tokens; sort by location instead.
    result["__begin"] = result[output_col_name].values.begin
    return result.sort_values("__begin")[[output_col_name]]

    Identify all matches of a dictionary on a sequence of tokens.

    :param tokens: `SpanArray` of token information, optionally wrapped in a
     `pd.Series`.
    :param dictionary: The dictionary to match, encoded as a `pd.DataFrame` in
     the format returned by `load_dict()`
    :param output_col_name: (optional) name of column of matching spans in the
     returned DataFrame

    :return: a single-column DataFrame of token ID spans of dictionary matches
    
Identify all matches of a dictionary on a sequence of tokens.
Identify all matches of a dictionary on a sequence of tokens .
def extract_dict(
    tokens: Union[SpanArray, pd.Series],
    dictionary: pd.DataFrame,
    output_col_name: str = "match",
):
    if isinstance(tokens, SpanArray):
        tokens = pd.Series(tokens)
    toks_tmp = pd.DataFrame(
        {
            "token_id": tokens.index,
            "normalized_text": tokens.array.normalized_covered_text,
        }
    )
    matches = pd.merge(
        dictionary, toks_tmp, left_on="toks_0", right_on="normalized_text"
    )
    matches.rename(columns={"token_id": "begin_token_id"}, inplace=True)
    matches_col_names = list(matches.columns)  
    begins_list = []
    ends_list = []
    max_entry_len = len(dictionary.columns)
    for match_len in range(1, max_entry_len):
        match_locs = pd.isna(matches["toks_{}".format(match_len)])
        match_begins = matches[match_locs]["begin_token_id"].to_numpy()
        match_ends = match_begins + match_len
        begins_list.append(match_begins)
        ends_list.append(match_ends)
        potential_matches = matches[~match_locs].copy()
        potential_matches.drop("normalized_text", axis=1, inplace=True)
        potential_matches["next_token_id"] = (
            potential_matches["begin_token_id"] + match_len
        )
        potential_matches = pd.merge(
            potential_matches, toks_tmp, left_on="next_token_id", right_on="token_id"
        )
        potential_matches = potential_matches[
            potential_matches["normalized_text"]
            == potential_matches["toks_{}".format(match_len)]
        ]
        matches = potential_matches[matches_col_names]
    begins = np.concatenate(begins_list)
    ends = np.concatenate(ends_list)
    result = pd.DataFrame(
        {output_col_name: TokenSpanArray(tokens.values, begins, ends)}
    )
    result["__begin"] = result[output_col_name].values.begin
    return result.sort_values("__begin")[[output_col_name]]
def extract_dict ( tokens : Union [ SpanArray , pd . Series ] , dictionary : pd . DataFrame , output_col_name : str = "match" , ) : if isinstance ( tokens , SpanArray ) : tokens = pd . Series ( tokens ) toks_tmp = pd . DataFrame ( { "token_id" : tokens . index , "normalized_text" : tokens . array . normalized_covered_text , } ) matches = pd . merge ( dictionary , toks_tmp , left_on = "toks_0" , right_on = "normalized_text" ) matches . rename ( columns = { "token_id" : "begin_token_id" } , inplace = True ) matches_col_names = list ( matches . columns ) begins_list = [ ] ends_list = [ ] max_entry_len = len ( dictionary . columns ) for match_len in range ( 1 , max_entry_len ) : match_locs = pd . isna ( matches [ "toks_{}" . format ( match_len ) ] ) match_begins = matches [ match_locs ] [ "begin_token_id" ] . to_numpy ( ) match_ends = match_begins + match_len begins_list . append ( match_begins ) ends_list . append ( match_ends ) potential_matches = matches [ ~ match_locs ] . copy ( ) potential_matches . drop ( "normalized_text" , axis = 1 , inplace = True ) potential_matches [ "next_token_id" ] = ( potential_matches [ "begin_token_id" ] + match_len ) potential_matches = pd . merge ( potential_matches , toks_tmp , left_on = "next_token_id" , right_on = "token_id" ) potential_matches = potential_matches [ potential_matches [ "normalized_text" ] == potential_matches [ "toks_{}" . format ( match_len ) ] ] matches = potential_matches [ matches_col_names ] begins = np . concatenate ( begins_list ) ends = np . concatenate ( ends_list ) result = pd . DataFrame ( { output_col_name : TokenSpanArray ( tokens . values , begins , ends ) } ) result [ "__begin" ] = result [ output_col_name ] . values . begin return result . sort_values ( "__begin" ) [ [ output_col_name ] ]
Identify all matches of a dictionary on a sequence of tokens.
Identify all matches of a dictionary on a sequence of tokens .
"""
    Identify all matches of a dictionary on a sequence of tokens.

    :param tokens: `SpanArray` of token information, optionally wrapped in a
     `pd.Series`.
    :param dictionary: The dictionary to match, encoded as a `pd.DataFrame` in
     the format returned by `load_dict()`
    :param output_col_name: (optional) name of column of matching spans in the
     returned DataFrame

    :return: a single-column DataFrame of token ID spans of dictionary matches
    """ # Box tokens into a pd.Series if not already boxed. # Wrap the important parts of the tokens series in a temporary dataframe. # noinspection PyUnresolvedReferences # Start by matching the first token. # We'll need this later # Check against remaining elements of matching dictionary entries and # accumulate the full set of matches as a list of IntervalIndexes # print("Match len: {}".format(match_len)) # Find matches of length match_len. Dictionary entries of this length # will have None in the column "toks_<match_len>". # print("Completed matches:\n{}".format(matches[match_locs])) # For the remaining partial matches against longer dictionary entries, # check the next token by merging with the tokens dataframe. # print("Raw potential matches:\n{}".format(potential_matches)) # print("Filtered potential matches:\n{}".format(potential_matches)) # The result of the join has some extra columns that we don't need. # Gather together all the sets of matches and wrap in a dataframe. # Results are sorted by number of tokens; sort by location instead.
a single-column DataFrame of token ID spans of dictionary matches
a single - column DataFrame of token ID spans of dictionary matches
tokens
Union[SpanArray, pd.Series]
`SpanArray` of token information, optionally wrapped in a
`pd.Series`.
` SpanArray ` of token information optionally wrapped in a ` pd . Series ` .
dictionary
pd.DataFrame
The dictionary to match, encoded as a `pd.DataFrame` in
the format returned by `load_dict()`
The dictionary to match encoded as a ` pd . DataFrame ` in the format returned by ` load_dict () `
output_col_name
str
(optional) name of column of matching spans in the
returned DataFrame
( optional ) name of column of matching spans in the returned DataFrame