pub fn encode<'s, E>(&self, input: E, add_special_tokens: bool) -> Result<Encoding>
    where
        E: Into<EncodeInput<'s>>,
    {
        // Extract sequences from the EncodeInput
        let (sequence, pair) = match input.into() {
            EncodeInput::Single(s1) => (s1, None),
            EncodeInput::Dual(s1, s2) => (s1, Some(s2)),
        };

        // Encode each sequence
        let encoding = self.encode_single_sequence(sequence, 0, OffsetType::Byte)?;
        let pair_encoding = pair
            .map(|sequence| self.encode_single_sequence(sequence, 1, OffsetType::Byte))
            .transpose()?;

        // And finally post process
        self.post_process(encoding, pair_encoding, add_special_tokens)
    }
/// Encode the given input. This method accepts both single sequences, as well as pair
/// sequences. Also, a sequence can be a string, or already pre-tokenized input directly:
///
/// ```
/// # use tokenizers::Tokenizer;
/// # use tokenizers::models::bpe::BPE;
/// # let mut tokenizer = Tokenizer::new(BPE::default());
/// #
/// // Sequences:
/// tokenizer.encode("Single sequence", false);
/// tokenizer.encode(("Sequence A", "Sequence B"), false);
///
/// // Pre-tokenized sequences:
/// tokenizer.encode(&["Single", "sequence"][..], false);
/// tokenizer.encode((
///     &["Sequence", "A"][..],
///     &["Sequence", "B"][..]
/// ), false);
///
/// // or even both types together:
/// tokenizer.encode(("A complete sequence", &["And", "a", "tokenized"][..]), false);
/// ```
Encode the given input. This method accepts both single sequences, as well as pair
sequences. Also, a sequence can be a string, or already pre-tokenized input directly.


Encode the given input . This method accepts both single sequences as well as pair sequences . Also a sequence can be a string or already pre - tokenized input directly .
pub fn encode<'s, E>(&self, input: E, add_special_tokens: bool) -> Result<Encoding>
    where
        E: Into<EncodeInput<'s>>,
    {
        // Extract sequences from the EncodeInput
        let (sequence, pair) = match input.into() {
            EncodeInput::Single(s1) => (s1, None),
            EncodeInput::Dual(s1, s2) => (s1, Some(s2)),
        };
        // Encode each sequence
        let encoding = self.encode_single_sequence(sequence, 0, OffsetType::Byte)?;
        let pair_encoding = pair
            .map(|sequence| self.encode_single_sequence(sequence, 1, OffsetType::Byte))
            .transpose()?;
        // And finally post process
        self.post_process(encoding, pair_encoding, add_special_tokens)
    }
pub fn encode < ' s , E > ( & self , input : E , add_special_tokens : bool ) -> Result < Encoding > where E : Into < EncodeInput < ' s > > , { // Extract sequences from the EncodeInput let ( sequence , pair ) = match input . into ( ) { EncodeInput :: Single ( s1 ) => ( s1 , None ) , EncodeInput :: Dual ( s1 , s2 ) => ( s1 , Some ( s2 ) ) , } ; // Encode each sequence let encoding = self . encode_single_sequence ( sequence , 0 , OffsetType :: Byte ) ? ; let pair_encoding = pair . map ( | sequence | self . encode_single_sequence ( sequence , 1 , OffsetType :: Byte ) ) . transpose ( ) ? ; // And finally post process self . post_process ( encoding , pair_encoding , add_special_tokens ) }
Encode the given input.
Encode the given input .
&self
input
E
add_special_tokens
bool
use tokenizers::tokenizer;
None
use tokenizers::models::bpe::bpe;
None
let mut tokenizer = tokenizer::new(bpe::default());
tokenizer.encode("Single sequence", false);
tokenizer.encode(("Sequence A", "Sequence B"), false).



or even both types together:
tokenizer.encode(("A complete sequence", &["And", "a", "tokenized"][..]), false);
```
tokenizer . encode ( " Single sequence " false ) ; tokenizer . encode (( " Sequence A " " Sequence B " ) false ) . or even both types together : tokenizer . encode (( " A complete sequence " & [ " And " " a " " tokenized " ] [ .. ] ) false ) ; ` ` `