text_extensions_for_pandas/spanner/join.py
def overlap_join(
    first_series: pd.Series,
    second_series: pd.Series,
    first_name: str = "first",
    second_name: str = "second",
):
    """
    Compute the join of two series of spans, where a pair of spans is
    considered to match if they overlap.

    :param first_series: First set of spans to join, wrapped in a `pd.Series`
    :param second_series: Second set of spans to join.
    :param first_name: Name to give the column in the returned dataframe that
     is derived from `first_series`.
    :param second_name: Column name for spans from `second_series` in the
     returned DataFrame.

    :returns: a new `pd.DataFrame` containing all pairs of spans that match
     the join predicate. Columns of the DataFrame will be named according
     to the `first_name` and `second_name` arguments.
    """
    # Python type checking doesn't enforce Pandas series dtypes.
    if (not isinstance(first_series.dtype, (SpanDtype, TokenSpanDtype))
            or not isinstance(second_series.dtype, (SpanDtype, TokenSpanDtype))):
        raise ValueError(f"Series must be of dtype {SpanDtype.name} "
                         f"or {TokenSpanDtype.name}. Dtypes received were "
                         f"{first_series.dtype} and {second_series.dtype}")

    # For now we always use character offsets.
    # TODO: Use token offsets of both sides of the join are TokenSpanArrays
    def _get_char_offsets(s: pd.Series):
        # noinspection PyUnresolvedReferences
        return s.array.begin, s.array.end

    first_begins, first_ends = _get_char_offsets(first_series)
    second_begins, second_ends = _get_char_offsets(second_series)

    # The algorithm here is what is known in the ER literature as "blocking".
    # First evaluate a looser predicate that can be translated to an equijoin,
    # then filter using the actual join predicate.

    # Compute average span length to determine blocking factor
    # TODO: Is average the right aggregate to use here?
    total_len = np.sum(first_ends - first_begins) + np.sum(second_ends - second_begins)
    num_spans = len(first_series.index) + len(second_series.index)
    average_len = 0. if 0 == num_spans else total_len / num_spans
    blocking_factor = max(1, int(np.floor(average_len)))

    # Generate a table of which blocks each row of the input participates in.
    # Use primary key (index) values because inputs can have duplicate spans.
    def _make_table(name, index, begins, ends):
        # TODO: Vectorize this part.
        indexes = []
        blocks = []
        for i, b, e in zip(index, begins, ends):
            for block in range(b // blocking_factor, e // blocking_factor + 1):
                indexes.append(i)
                blocks.append(block)
        return pd.DataFrame({name: indexes, "block": blocks})

    first_table = _make_table("first", first_series.index, first_begins, first_ends)
    second_table = _make_table(
        "second", second_series.index, second_begins, second_ends
    )

    # Do an equijoin on block ID and remove duplicates from the resulting
    # <first key, second key> relation.
    merged_table = pd.merge(first_table, second_table)
    key_pairs = merged_table.groupby(["first", "second"]).aggregate(
        {"first": "first", "second": "first"}
    )

    # Join the keys back with the original series to form the result, plus
    # some extra values due to blocking.
    block_result = pd.DataFrame(
        {
            first_name: first_series.loc[key_pairs["first"]].array,
            second_name: second_series.loc[key_pairs["second"]].array,
        }
    )

    # Filter out extra values from blocking
    mask = block_result[first_name].array.overlaps(block_result[second_name].array)
    return block_result[mask].reset_index(drop=True)

    Compute the join of two series of spans, where a pair of spans is
    considered to match if they overlap.

    :param first_series: First set of spans to join, wrapped in a `pd.Series`
    :param second_series: Second set of spans to join.
    :param first_name: Name to give the column in the returned dataframe that
     is derived from `first_series`.
    :param second_name: Column name for spans from `second_series` in the
     returned DataFrame.

    :returns: a new `pd.DataFrame` containing all pairs of spans that match
     the join predicate. Columns of the DataFrame will be named according
     to the `first_name` and `second_name` arguments.
    
Compute the join of two series of spans, where a pair of spans is
considered to match if they overlap.
Compute the join of two series of spans where a pair of spans is considered to match if they overlap .
def overlap_join(
    first_series: pd.Series,
    second_series: pd.Series,
    first_name: str = "first",
    second_name: str = "second",
):
    if (not isinstance(first_series.dtype, (SpanDtype, TokenSpanDtype))
            or not isinstance(second_series.dtype, (SpanDtype, TokenSpanDtype))):
        raise ValueError(f"Series must be of dtype {SpanDtype.name} "
                         f"or {TokenSpanDtype.name}. Dtypes received were "
                         f"{first_series.dtype} and {second_series.dtype}")
    def _get_char_offsets(s: pd.Series):
        return s.array.begin, s.array.end
    first_begins, first_ends = _get_char_offsets(first_series)
    second_begins, second_ends = _get_char_offsets(second_series)
    total_len = np.sum(first_ends - first_begins) + np.sum(second_ends - second_begins)
    num_spans = len(first_series.index) + len(second_series.index)
    average_len = 0. if 0 == num_spans else total_len / num_spans
    blocking_factor = max(1, int(np.floor(average_len)))
    def _make_table(name, index, begins, ends):
        indexes = []
        blocks = []
        for i, b, e in zip(index, begins, ends):
            for block in range(b // blocking_factor, e // blocking_factor + 1):
                indexes.append(i)
                blocks.append(block)
        return pd.DataFrame({name: indexes, "block": blocks})
    first_table = _make_table("first", first_series.index, first_begins, first_ends)
    second_table = _make_table(
        "second", second_series.index, second_begins, second_ends
    )
    merged_table = pd.merge(first_table, second_table)
    key_pairs = merged_table.groupby(["first", "second"]).aggregate(
        {"first": "first", "second": "first"}
    )
    block_result = pd.DataFrame(
        {
            first_name: first_series.loc[key_pairs["first"]].array,
            second_name: second_series.loc[key_pairs["second"]].array,
        }
    )
    mask = block_result[first_name].array.overlaps(block_result[second_name].array)
    return block_result[mask].reset_index(drop=True)
def overlap_join ( first_series : pd . Series , second_series : pd . Series , first_name : str = "first" , second_name : str = "second" , ) : if ( not isinstance ( first_series . dtype , ( SpanDtype , TokenSpanDtype ) ) or not isinstance ( second_series . dtype , ( SpanDtype , TokenSpanDtype ) ) ) : raise ValueError ( f"Series must be of dtype {SpanDtype.name} " f"or {TokenSpanDtype.name}. Dtypes received were " f"{first_series.dtype} and {second_series.dtype}" ) def _get_char_offsets ( s : pd . Series ) : return s . array . begin , s . array . end first_begins , first_ends = _get_char_offsets ( first_series ) second_begins , second_ends = _get_char_offsets ( second_series ) total_len = np . sum ( first_ends - first_begins ) + np . sum ( second_ends - second_begins ) num_spans = len ( first_series . index ) + len ( second_series . index ) average_len = 0. if 0 == num_spans else total_len / num_spans blocking_factor = max ( 1 , int ( np . floor ( average_len ) ) ) def _make_table ( name , index , begins , ends ) : indexes = [ ] blocks = [ ] for i , b , e in zip ( index , begins , ends ) : for block in range ( b // blocking_factor , e // blocking_factor + 1 ) : indexes . append ( i ) blocks . append ( block ) return pd . DataFrame ( { name : indexes , "block" : blocks } ) first_table = _make_table ( "first" , first_series . index , first_begins , first_ends ) second_table = _make_table ( "second" , second_series . index , second_begins , second_ends ) merged_table = pd . merge ( first_table , second_table ) key_pairs = merged_table . groupby ( [ "first" , "second" ] ) . aggregate ( { "first" : "first" , "second" : "first" } ) block_result = pd . DataFrame ( { first_name : first_series . loc [ key_pairs [ "first" ] ] . array , second_name : second_series . loc [ key_pairs [ "second" ] ] . array , } ) mask = block_result [ first_name ] . array . overlaps ( block_result [ second_name ] . array ) return block_result [ mask ] . reset_index ( drop = True )
Compute the join of two series of spans, where a pair of spans is
considered to match if they overlap.
Compute the join of two series of spans where a pair of spans is considered to match if they overlap .
"""
    Compute the join of two series of spans, where a pair of spans is
    considered to match if they overlap.

    :param first_series: First set of spans to join, wrapped in a `pd.Series`
    :param second_series: Second set of spans to join.
    :param first_name: Name to give the column in the returned dataframe that
     is derived from `first_series`.
    :param second_name: Column name for spans from `second_series` in the
     returned DataFrame.

    :returns: a new `pd.DataFrame` containing all pairs of spans that match
     the join predicate. Columns of the DataFrame will be named according
     to the `first_name` and `second_name` arguments.
    """ # Python type checking doesn't enforce Pandas series dtypes. # For now we always use character offsets. # TODO: Use token offsets of both sides of the join are TokenSpanArrays # noinspection PyUnresolvedReferences # The algorithm here is what is known in the ER literature as "blocking". # First evaluate a looser predicate that can be translated to an equijoin, # then filter using the actual join predicate. # Compute average span length to determine blocking factor # TODO: Is average the right aggregate to use here? # Generate a table of which blocks each row of the input participates in. # Use primary key (index) values because inputs can have duplicate spans. # TODO: Vectorize this part. # Do an equijoin on block ID and remove duplicates from the resulting # <first key, second key> relation. # Join the keys back with the original series to form the result, plus # some extra values due to blocking. # Filter out extra values from blocking
a new `pd.DataFrame` containing all pairs of spans that match
the join predicate. Columns of the DataFrame will be named according
to the `first_name` and `second_name` arguments.
a new ` pd . DataFrame ` containing all pairs of spans that match the join predicate . Columns of the DataFrame will be named according to the ` first_name ` and ` second_name ` arguments .
first_series
pd.Series
First set of spans to join, wrapped in a `pd.Series`
First set of spans to join wrapped in a ` pd . Series `
second_series
pd.Series
Second set of spans to join.
Second set of spans to join .
first_name
str
Name to give the column in the returned dataframe that
is derived from `first_series`.
Name to give the column in the returned dataframe that is derived from ` first_series ` .
second_name
str
Column name for spans from `second_series` in the
returned DataFrame.
Column name for spans from ` second_series ` in the returned DataFrame .