text_extensions_for_pandas/io/bert.py
def seq_to_windows(
    seq: np.ndarray, overlap: int, non_overlap: int
) -> Dict[str, np.ndarray]:
    """
    Convert a variable-length sequence into a set of fixed length windows,
    adding padding as necessary.

    Usually this function is used to prepare batches of BERT tokens to feed
    to a BERT model.

    :param seq: Original variable length sequence, as a 1D numpy array
    :param overlap: How much overlap there should be between adjacent windows
    :param non_overlap: How much non-overlapping content between the overlapping
     regions there should be at the middle of each window?

    :returns: Dictionary with the keys "input_ids" and "attention_masks" mapped
              to NumPy arrays as described below.

              * "input_ids" (where ``d`` is the returned dictionary):
                2D ``np.ndarray`` of fixed-length windows
              * "attention_masks": 2D ``np.ndarray`` of attention masks (1 for
                tokens that are NOT masked, 0 for tokens that are masked)
                to feed into your favorite BERT-like embedding generator.


    """
    if len(seq.shape) != 1:
        raise ValueError(f"Input array must be 1D; got shape {seq.shape}")
    window_length, pre_padding, post_padding = _compute_padding(
        len(seq), overlap, non_overlap
    )

    # First generate the windows as a padded flat arrays.
    padded_length = len(seq) + pre_padding + post_padding
    buf = np.zeros(shape=[padded_length], dtype=seq.dtype)
    buf[pre_padding : pre_padding + len(seq)] = seq

    mask_buf = np.zeros_like(buf, dtype=int)  # 0 == MASKED
    mask_buf[pre_padding : pre_padding + len(seq)] = 1  # 1 == NOT MASKED

    # Reshape the flat arrays into overlapping windows.
    num_windows = padded_length // (overlap + non_overlap)
    windows = np.zeros(shape=[num_windows, window_length], dtype=seq.dtype)
    masks = np.zeros(shape=[num_windows, window_length], dtype=int)
    for i in range(num_windows):
        start = i * (overlap + non_overlap)
        windows[i] = buf[start : start + window_length]
        masks[i] = mask_buf[start : start + window_length]
    return {"input_ids": windows, "attention_masks": masks}

    Convert a variable-length sequence into a set of fixed length windows,
    adding padding as necessary.

    Usually this function is used to prepare batches of BERT tokens to feed
    to a BERT model.

    :param seq: Original variable length sequence, as a 1D numpy array
    :param overlap: How much overlap there should be between adjacent windows
    :param non_overlap: How much non-overlapping content between the overlapping
     regions there should be at the middle of each window?

    :returns: Dictionary with the keys "input_ids" and "attention_masks" mapped
              to NumPy arrays as described below.

              * "input_ids" (where ``d`` is the returned dictionary):
                2D ``np.ndarray`` of fixed-length windows
              * "attention_masks": 2D ``np.ndarray`` of attention masks (1 for
                tokens that are NOT masked, 0 for tokens that are masked)
                to feed into your favorite BERT-like embedding generator.


    
Convert a variable-length sequence into a set of fixed length windows,
adding padding as necessary.

Usually this function is used to prepare batches of BERT tokens to feed
to a BERT model.
Convert a variable - length sequence into a set of fixed length windows adding padding as necessary . Usually this function is used to prepare batches of BERT tokens to feed to a BERT model .
def seq_to_windows(
    seq: np.ndarray, overlap: int, non_overlap: int
) -> Dict[str, np.ndarray]:
    if len(seq.shape) != 1:
        raise ValueError(f"Input array must be 1D; got shape {seq.shape}")
    window_length, pre_padding, post_padding = _compute_padding(
        len(seq), overlap, non_overlap
    )
    padded_length = len(seq) + pre_padding + post_padding
    buf = np.zeros(shape=[padded_length], dtype=seq.dtype)
    buf[pre_padding : pre_padding + len(seq)] = seq
    mask_buf = np.zeros_like(buf, dtype=int)  
    mask_buf[pre_padding : pre_padding + len(seq)] = 1  
    num_windows = padded_length // (overlap + non_overlap)
    windows = np.zeros(shape=[num_windows, window_length], dtype=seq.dtype)
    masks = np.zeros(shape=[num_windows, window_length], dtype=int)
    for i in range(num_windows):
        start = i * (overlap + non_overlap)
        windows[i] = buf[start : start + window_length]
        masks[i] = mask_buf[start : start + window_length]
    return {"input_ids": windows, "attention_masks": masks}
def seq_to_windows ( seq : np . ndarray , overlap : int , non_overlap : int ) -> Dict [ str , np . ndarray ] : if len ( seq . shape ) != 1 : raise ValueError ( f"Input array must be 1D; got shape {seq.shape}" ) window_length , pre_padding , post_padding = _compute_padding ( len ( seq ) , overlap , non_overlap ) padded_length = len ( seq ) + pre_padding + post_padding buf = np . zeros ( shape = [ padded_length ] , dtype = seq . dtype ) buf [ pre_padding : pre_padding + len ( seq ) ] = seq mask_buf = np . zeros_like ( buf , dtype = int ) mask_buf [ pre_padding : pre_padding + len ( seq ) ] = 1 num_windows = padded_length // ( overlap + non_overlap ) windows = np . zeros ( shape = [ num_windows , window_length ] , dtype = seq . dtype ) masks = np . zeros ( shape = [ num_windows , window_length ] , dtype = int ) for i in range ( num_windows ) : start = i * ( overlap + non_overlap ) windows [ i ] = buf [ start : start + window_length ] masks [ i ] = mask_buf [ start : start + window_length ] return { "input_ids" : windows , "attention_masks" : masks }
Convert a variable-length sequence into a set of fixed length windows,
adding padding as necessary.
Convert a variable - length sequence into a set of fixed length windows adding padding as necessary .
"""
    Convert a variable-length sequence into a set of fixed length windows,
    adding padding as necessary.

    Usually this function is used to prepare batches of BERT tokens to feed
    to a BERT model.

    :param seq: Original variable length sequence, as a 1D numpy array
    :param overlap: How much overlap there should be between adjacent windows
    :param non_overlap: How much non-overlapping content between the overlapping
     regions there should be at the middle of each window?

    :returns: Dictionary with the keys "input_ids" and "attention_masks" mapped
              to NumPy arrays as described below.

              * "input_ids" (where ``d`` is the returned dictionary):
                2D ``np.ndarray`` of fixed-length windows
              * "attention_masks": 2D ``np.ndarray`` of attention masks (1 for
                tokens that are NOT masked, 0 for tokens that are masked)
                to feed into your favorite BERT-like embedding generator.


    """ # First generate the windows as a padded flat arrays. # 0 == MASKED # 1 == NOT MASKED # Reshape the flat arrays into overlapping windows.
Dictionary with the keys "input_ids" and "attention_masks" mapped
to NumPy arrays as described below.

"input_ids" (where ``d`` is the returned dictionary):
2D ``np.ndarray`` of fixed-length windows
"attention_masks": 2D ``np.ndarray`` of attention masks (1 for
tokens that are NOT masked, 0 for tokens that are masked)
to feed into your favorite BERT-like embedding generator.
Dictionary with the keys " input_ids " and " attention_masks " mapped to NumPy arrays as described below . " input_ids " ( where ` ` d ` ` is the returned dictionary ) : 2D ` ` np . ndarray ` ` of fixed - length windows " attention_masks " : 2D ` ` np . ndarray ` ` of attention masks ( 1 for tokens that are NOT masked 0 for tokens that are masked ) to feed into your favorite BERT - like embedding generator .
seq
np.ndarray
Original variable length sequence, as a 1D numpy array
Original variable length sequence as a 1D numpy array
overlap
int
None
non_overlap
int
None