text_extensions_for_pandas/io/bert.py
def add_embeddings(
    df: pd.DataFrame, bert: Any, overlap: int = 32, non_overlap: int = 64
) -> pd.DataFrame:
    """
    Add BERT embeddings to a DataFrame of BERT tokens.

    :param df: DataFrame containing BERT tokens, as returned by
      :func:`make_bert_tokens` Must contain a column
      ``input_id`` containing token IDs.
    :param bert: PyTorch-based BERT model from the ``transformers`` library
    :param overlap: (optional) how much overlap there should be between adjacent windows
    :param non_overlap: (optional) how much non-overlapping content between the
     overlapping regions there should be at the middle of each window?
    :returns: A copy of ``df`` with a new column, "embedding", containing
     BERT embeddings as a ``TensorArray``.

    .. note:: PyTorch must be installed to run this function.
    """
    # Import torch inline so that the rest of this library will function without it.
    # noinspection PyPackageRequirements
    import torch

    flat_input_ids = df["input_id"].values
    windows = seq_to_windows(flat_input_ids, overlap, non_overlap)
    bert_result = bert(
        input_ids=torch.tensor(windows["input_ids"]),
        attention_mask=torch.tensor(windows["attention_masks"]),
    )
    hidden_states = windows_to_seq(
        flat_input_ids, bert_result[0].detach().numpy(), overlap, non_overlap
    )
    embeddings = TensorArray(hidden_states)
    ret = df.copy()
    ret["embedding"] = embeddings
    return ret

    Add BERT embeddings to a DataFrame of BERT tokens.

    :param df: DataFrame containing BERT tokens, as returned by
      :func:`make_bert_tokens` Must contain a column
      ``input_id`` containing token IDs.
    :param bert: PyTorch-based BERT model from the ``transformers`` library
    :param overlap: (optional) how much overlap there should be between adjacent windows
    :param non_overlap: (optional) how much non-overlapping content between the
     overlapping regions there should be at the middle of each window?
    :returns: A copy of ``df`` with a new column, "embedding", containing
     BERT embeddings as a ``TensorArray``.

    .. note:: PyTorch must be installed to run this function.
    
Add BERT embeddings to a DataFrame of BERT tokens.
Add BERT embeddings to a DataFrame of BERT tokens .
def add_embeddings(
    df: pd.DataFrame, bert: Any, overlap: int = 32, non_overlap: int = 64
) -> pd.DataFrame:
    import torch
    flat_input_ids = df["input_id"].values
    windows = seq_to_windows(flat_input_ids, overlap, non_overlap)
    bert_result = bert(
        input_ids=torch.tensor(windows["input_ids"]),
        attention_mask=torch.tensor(windows["attention_masks"]),
    )
    hidden_states = windows_to_seq(
        flat_input_ids, bert_result[0].detach().numpy(), overlap, non_overlap
    )
    embeddings = TensorArray(hidden_states)
    ret = df.copy()
    ret["embedding"] = embeddings
    return ret
def add_embeddings ( df : pd . DataFrame , bert : Any , overlap : int = 32 , non_overlap : int = 64 ) -> pd . DataFrame : import torch flat_input_ids = df [ "input_id" ] . values windows = seq_to_windows ( flat_input_ids , overlap , non_overlap ) bert_result = bert ( input_ids = torch . tensor ( windows [ "input_ids" ] ) , attention_mask = torch . tensor ( windows [ "attention_masks" ] ) , ) hidden_states = windows_to_seq ( flat_input_ids , bert_result [ 0 ] . detach ( ) . numpy ( ) , overlap , non_overlap ) embeddings = TensorArray ( hidden_states ) ret = df . copy ( ) ret [ "embedding" ] = embeddings return ret
Add BERT embeddings to a DataFrame of BERT tokens.
Add BERT embeddings to a DataFrame of BERT tokens .
"""
    Add BERT embeddings to a DataFrame of BERT tokens.

    :param df: DataFrame containing BERT tokens, as returned by
      :func:`make_bert_tokens` Must contain a column
      ``input_id`` containing token IDs.
    :param bert: PyTorch-based BERT model from the ``transformers`` library
    :param overlap: (optional) how much overlap there should be between adjacent windows
    :param non_overlap: (optional) how much non-overlapping content between the
     overlapping regions there should be at the middle of each window?
    :returns: A copy of ``df`` with a new column, "embedding", containing
     BERT embeddings as a ``TensorArray``.

    .. note:: PyTorch must be installed to run this function.
    """ # Import torch inline so that the rest of this library will function without it. # noinspection PyPackageRequirements
A copy of ``df`` with a new column, "embedding", containing
BERT embeddings as a ``TensorArray``.

: PyTorch must be installed to run this function.
A copy of ` ` df ` ` with a new column " embedding " containing BERT embeddings as a ` ` TensorArray ` ` . : PyTorch must be installed to run this function .
df
pd.DataFrame
DataFrame containing BERT tokens, as returned by
:func:`make_bert_tokens` Must contain a column
``input_id`` containing token IDs.
DataFrame containing BERT tokens as returned by : func : ` make_bert_tokens ` Must contain a column ` ` input_id ` ` containing token IDs .
bert
Any
PyTorch-based BERT model from the ``transformers`` library
PyTorch - based BERT model from the ` ` transformers ` ` library
overlap
int
(optional) how much overlap there should be between adjacent windows
( optional ) how much overlap there should be between adjacent windows
non_overlap
int
None