text_extensions_for_pandas/array/token_span.py
def align_to_tokens(cls, tokens: Any, spans: Any):
        """
        Align a set of character or token-based spans to a specified
        tokenization, producing a `TokenSpanArray` of token-based spans.

        :param tokens: The tokens to align to, as any type that
         :func:`SpanArray.make_array` accepts.
        :param spans: The spans to align. These spans must all target the same text
         as ``tokens``.

        :return: An array of :class:`TokenSpan` objects aligned to the tokens of
            ``tokens``.
            Raises :class:`ValueError` if any of the spans in ``spans`` doesn't start and
            end on a token boundary.
        """
        tokens = SpanArray.make_array(tokens)
        spans = SpanArray.make_array(spans)

        if not tokens.is_single_document:
            raise ValueError(f"Tokens cover more than one document (tokens are {tokens})")
        if not spans.is_single_document:
            raise ValueError(f"Spans cover more than one document (spans are {spans})")

        # Create and join temporary dataframes
        tokens_df = pd.DataFrame({
            "token_index": np.arange(len(tokens)),
            "token_begin": tokens.begin,
            "token_end": tokens.end
        })
        spans_df = pd.DataFrame({
            "span_index": np.arange(len(spans)),
            "span_begin": spans.begin,
            "span_end": spans.end
        })

        # Ignore zero-length tokens
        # TODO: Is this the right thing to do?
        tokens_df = tokens_df[tokens_df["token_begin"] != tokens_df["token_end"]]

        begin_matches = pd.merge(tokens_df, spans_df,
                                 left_on="token_begin",
                                 right_on="span_begin",
                                 how="right", indicator=True)

        mismatched = begin_matches[begin_matches["_merge"] == "right_only"]
        if len(mismatched.index) > 0:
            raise ValueError(
                f"The following span(s) did not align with the begin offset\n"
                f"of any token:\n"
                f"{mismatched[['span_index', 'span_begin', 'span_end']]}")

        end_matches = pd.merge(tokens_df, spans_df,
                               left_on="token_end",
                               right_on="span_end",
                               how="right", indicator=True)

        mismatched = end_matches[end_matches["_merge"] == "right_only"]
        if len(mismatched.index) > 0:
            raise ValueError(
                f"The following span(s) did not align with the end offset\n"
                f"of any token:\n"
                f"{mismatched[['span_index', 'span_begin', 'span_end']]}")

        # Join on span index to get (begin, end) pairs.
        begins_and_ends = pd.merge(
            begin_matches[["token_index", "span_index"]],
            end_matches[["token_index", "span_index"]],
            on="span_index", suffixes=("_begin", "_end"),
            sort=True)

        return TokenSpanArray(tokens,
                              begins_and_ends["token_index_begin"],
                              begins_and_ends["token_index_end"] + 1)

        Align a set of character or token-based spans to a specified
        tokenization, producing a `TokenSpanArray` of token-based spans.

        :param tokens: The tokens to align to, as any type that
         :func:`SpanArray.make_array` accepts.
        :param spans: The spans to align. These spans must all target the same text
         as ``tokens``.

        :return: An array of :class:`TokenSpan` objects aligned to the tokens of
            ``tokens``.
            Raises :class:`ValueError` if any of the spans in ``spans`` doesn't start and
            end on a token boundary.
        
Align a set of character or token-based spans to a specified
tokenization, producing a `TokenSpanArray` of token-based spans.
Align a set of character or token - based spans to a specified tokenization producing a ` TokenSpanArray ` of token - based spans .
def align_to_tokens(cls, tokens: Any, spans: Any):
        tokens = SpanArray.make_array(tokens)
        spans = SpanArray.make_array(spans)
        if not tokens.is_single_document:
            raise ValueError(f"Tokens cover more than one document (tokens are {tokens})")
        if not spans.is_single_document:
            raise ValueError(f"Spans cover more than one document (spans are {spans})")
        tokens_df = pd.DataFrame({
            "token_index": np.arange(len(tokens)),
            "token_begin": tokens.begin,
            "token_end": tokens.end
        })
        spans_df = pd.DataFrame({
            "span_index": np.arange(len(spans)),
            "span_begin": spans.begin,
            "span_end": spans.end
        })
        tokens_df = tokens_df[tokens_df["token_begin"] != tokens_df["token_end"]]
        begin_matches = pd.merge(tokens_df, spans_df,
                                 left_on="token_begin",
                                 right_on="span_begin",
                                 how="right", indicator=True)
        mismatched = begin_matches[begin_matches["_merge"] == "right_only"]
        if len(mismatched.index) > 0:
            raise ValueError(
                f"The following span(s) did not align with the begin offset\n"
                f"of any token:\n"
                f"{mismatched[['span_index', 'span_begin', 'span_end']]}")
        end_matches = pd.merge(tokens_df, spans_df,
                               left_on="token_end",
                               right_on="span_end",
                               how="right", indicator=True)
        mismatched = end_matches[end_matches["_merge"] == "right_only"]
        if len(mismatched.index) > 0:
            raise ValueError(
                f"The following span(s) did not align with the end offset\n"
                f"of any token:\n"
                f"{mismatched[['span_index', 'span_begin', 'span_end']]}")
        begins_and_ends = pd.merge(
            begin_matches[["token_index", "span_index"]],
            end_matches[["token_index", "span_index"]],
            on="span_index", suffixes=("_begin", "_end"),
            sort=True)
        return TokenSpanArray(tokens,
                              begins_and_ends["token_index_begin"],
                              begins_and_ends["token_index_end"] + 1)
def align_to_tokens ( cls , tokens : Any , spans : Any ) : tokens = SpanArray . make_array ( tokens ) spans = SpanArray . make_array ( spans ) if not tokens . is_single_document : raise ValueError ( f"Tokens cover more than one document (tokens are {tokens})" ) if not spans . is_single_document : raise ValueError ( f"Spans cover more than one document (spans are {spans})" ) tokens_df = pd . DataFrame ( { "token_index" : np . arange ( len ( tokens ) ) , "token_begin" : tokens . begin , "token_end" : tokens . end } ) spans_df = pd . DataFrame ( { "span_index" : np . arange ( len ( spans ) ) , "span_begin" : spans . begin , "span_end" : spans . end } ) tokens_df = tokens_df [ tokens_df [ "token_begin" ] != tokens_df [ "token_end" ] ] begin_matches = pd . merge ( tokens_df , spans_df , left_on = "token_begin" , right_on = "span_begin" , how = "right" , indicator = True ) mismatched = begin_matches [ begin_matches [ "_merge" ] == "right_only" ] if len ( mismatched . index ) > 0 : raise ValueError ( f"The following span(s) did not align with the begin offset\n" f"of any token:\n" f"{mismatched[['span_index', 'span_begin', 'span_end']]}" ) end_matches = pd . merge ( tokens_df , spans_df , left_on = "token_end" , right_on = "span_end" , how = "right" , indicator = True ) mismatched = end_matches [ end_matches [ "_merge" ] == "right_only" ] if len ( mismatched . index ) > 0 : raise ValueError ( f"The following span(s) did not align with the end offset\n" f"of any token:\n" f"{mismatched[['span_index', 'span_begin', 'span_end']]}" ) begins_and_ends = pd . merge ( begin_matches [ [ "token_index" , "span_index" ] ] , end_matches [ [ "token_index" , "span_index" ] ] , on = "span_index" , suffixes = ( "_begin" , "_end" ) , sort = True ) return TokenSpanArray ( tokens , begins_and_ends [ "token_index_begin" ] , begins_and_ends [ "token_index_end" ] + 1 )
Align a set of character or token-based spans to a specified
tokenization, producing a `TokenSpanArray` of token-based spans.
Align a set of character or token - based spans to a specified tokenization producing a ` TokenSpanArray ` of token - based spans .
"""
        Align a set of character or token-based spans to a specified
        tokenization, producing a `TokenSpanArray` of token-based spans.

        :param tokens: The tokens to align to, as any type that
         :func:`SpanArray.make_array` accepts.
        :param spans: The spans to align. These spans must all target the same text
         as ``tokens``.

        :return: An array of :class:`TokenSpan` objects aligned to the tokens of
            ``tokens``.
            Raises :class:`ValueError` if any of the spans in ``spans`` doesn't start and
            end on a token boundary.
        """ # Create and join temporary dataframes # Ignore zero-length tokens # TODO: Is this the right thing to do? # Join on span index to get (begin, end) pairs.
An array of :class:`TokenSpan` objects aligned to the tokens of
``tokens``.
Raises :class:`ValueError` if any of the spans in ``spans`` doesn't start and
end on a token boundary.
An array of : class : ` TokenSpan ` objects aligned to the tokens of ` ` tokens ` ` . Raises : class : ` ValueError ` if any of the spans in ` ` spans ` ` doesn ' t start and end on a token boundary .
cls
tokens
Any
The tokens to align to, as any type that
:func:`SpanArray.make_array` accepts.
The tokens to align to as any type that : func : ` SpanArray . make_array ` accepts .
spans
Any
The spans to align. These spans must all target the same text
as ``tokens``.
The spans to align . These spans must all target the same text as ` ` tokens ` ` .