pub fn encode_batch_char_offsets<'s, E>(
        &self,
        inputs: Vec<E>,
        add_special_tokens: bool,
    ) -> Result<Vec<Encoding>>
    where
        E: Into<EncodeInput<'s>> + Send,
    {
        let mut encodings = inputs
            .into_maybe_par_iter()
            .map(|input| self.encode_char_offsets(input, add_special_tokens))
            .collect::<Result<Vec<Encoding>>>()?;

        if let Some(params) = &self.padding {
            // We do the padding here to make sure we handle the batch padding
            pad_encodings(&mut encodings, params)?;
        }

        Ok(encodings)
    }
/// Encode all the sentences in parallel, using multiple threads.
/// The offsets on each `Encoding` will be relative to chars instead of bytes.
Encode all the sentences in parallel, using multiple threads.
The offsets on each `Encoding` will be relative to chars instead of bytes.
Encode all the sentences in parallel using multiple threads . The offsets on each ` Encoding ` will be relative to chars instead of bytes .
pub fn encode_batch_char_offsets<'s, E>(
        &self,
        inputs: Vec<E>,
        add_special_tokens: bool,
    ) -> Result<Vec<Encoding>>
    where
        E: Into<EncodeInput<'s>> + Send,
    {
        let mut encodings = inputs
            .into_maybe_par_iter()
            .map(|input| self.encode_char_offsets(input, add_special_tokens))
            .collect::<Result<Vec<Encoding>>>()?;
        if let Some(params) = &self.padding {
            // We do the padding here to make sure we handle the batch padding
            pad_encodings(&mut encodings, params)?;
        }
        Ok(encodings)
    }
pub fn encode_batch_char_offsets < ' s , E > ( & self , inputs : Vec < E > , add_special_tokens : bool , ) -> Result < Vec < Encoding > > where E : Into < EncodeInput < ' s > > + Send , { let mut encodings = inputs . into_maybe_par_iter ( ) . map ( | input | self . encode_char_offsets ( input , add_special_tokens ) ) . collect :: < Result < Vec < Encoding > > > ( ) ? ; if let Some ( params ) = & self . padding { // We do the padding here to make sure we handle the batch padding pad_encodings ( & mut encodings , params ) ? ; } Ok ( encodings ) }
Encode all the sentences in parallel, using multiple threads.
Encode all the sentences in parallel using multiple threads .
&self
inputs
Vec<E>
add_special_tokens
bool