pub fn post_process(
        &self,
        encoding: Encoding,
        pair_encoding: Option<Encoding>,
        add_special_tokens: bool,
    ) -> Result<Encoding> {
        // 1. First we truncate if needed
        let (encoding, pair_encoding) = {
            if let Some(trunc) = &self.truncation {
                let n_added_tokens = if let Some(processor) = &self.post_processor {
                    processor.added_tokens(pair_encoding.is_some())
                } else {
                    0
                };

                if add_special_tokens && n_added_tokens > 0 {
                    let params = TruncationParams {
                        max_length: trunc.max_length - n_added_tokens,
                        ..*trunc
                    };
                    truncate_encodings(encoding, pair_encoding, &params)?
                } else {
                    truncate_encodings(encoding, pair_encoding, trunc)?
                }
            } else {
                (encoding, pair_encoding)
            }
        };

        // 2. Then We post process
        let final_encoding = if let Some(processor) = &self.post_processor {
            processor.process(encoding, pair_encoding, add_special_tokens)?
        } else {
            <dyn PostProcessor>::default_process(encoding, pair_encoding, add_special_tokens)?
        };

        // 3. Then we pad if needed
        let [final_encoding] = if let Some(params) = &self.padding {
            let mut arr = [final_encoding];
            pad_encodings(&mut arr, params)?;
            arr
        } else {
            [final_encoding]
        };

        Ok(final_encoding)
    }
/// Post processing logic, handling the case where there is no PostProcessor set
Post processing logic, handling the case where there is no PostProcessor set
Post processing logic handling the case where there is no PostProcessor set
pub fn post_process(
        &self,
        encoding: Encoding,
        pair_encoding: Option<Encoding>,
        add_special_tokens: bool,
    ) -> Result<Encoding> {
        // 1. First we truncate if needed
        let (encoding, pair_encoding) = {
            if let Some(trunc) = &self.truncation {
                let n_added_tokens = if let Some(processor) = &self.post_processor {
                    processor.added_tokens(pair_encoding.is_some())
                } else {
                    0
                };
                if add_special_tokens && n_added_tokens > 0 {
                    let params = TruncationParams {
                        max_length: trunc.max_length - n_added_tokens,
                        ..*trunc
                    };
                    truncate_encodings(encoding, pair_encoding, &params)?
                } else {
                    truncate_encodings(encoding, pair_encoding, trunc)?
                }
            } else {
                (encoding, pair_encoding)
            }
        };
        // 2. Then We post process
        let final_encoding = if let Some(processor) = &self.post_processor {
            processor.process(encoding, pair_encoding, add_special_tokens)?
        } else {
            <dyn PostProcessor>::default_process(encoding, pair_encoding, add_special_tokens)?
        };
        // 3. Then we pad if needed
        let [final_encoding] = if let Some(params) = &self.padding {
            let mut arr = [final_encoding];
            pad_encodings(&mut arr, params)?;
            arr
        } else {
            [final_encoding]
        };
        Ok(final_encoding)
    }
pub fn post_process ( & self , encoding : Encoding , pair_encoding : Option < Encoding > , add_special_tokens : bool , ) -> Result < Encoding > { // 1. First we truncate if needed let ( encoding , pair_encoding ) = { if let Some ( trunc ) = & self . truncation { let n_added_tokens = if let Some ( processor ) = & self . post_processor { processor . added_tokens ( pair_encoding . is_some ( ) ) } else { 0 } ; if add_special_tokens && n_added_tokens > 0 { let params = TruncationParams { max_length : trunc . max_length - n_added_tokens , .. * trunc } ; truncate_encodings ( encoding , pair_encoding , & params ) ? } else { truncate_encodings ( encoding , pair_encoding , trunc ) ? } } else { ( encoding , pair_encoding ) } } ; // 2. Then We post process let final_encoding = if let Some ( processor ) = & self . post_processor { processor . process ( encoding , pair_encoding , add_special_tokens ) ? } else { < dyn PostProcessor > :: default_process ( encoding , pair_encoding , add_special_tokens ) ? } ; // 3. Then we pad if needed let [ final_encoding ] = if let Some ( params ) = & self . padding { let mut arr = [ final_encoding ] ; pad_encodings ( & mut arr , params ) ? ; arr } else { [ final_encoding ] } ; Ok ( final_encoding ) }
Post processing logic, handling the case where there is no PostProcessor set
Post processing logic handling the case where there is no PostProcessor set
&self
encoding
Encoding
pair_encoding
Option<Encoding>
add_special_tokens
bool