pub fn build_input_with_special_tokens(
        &self,
        tokens_1: Vec<i64>,
        tokens_2: Option<Vec<i64>>,
        offsets_1: Vec<Option<Offset>>,
        offsets_2: Option<Vec<Option<Offset>>>,
        original_offsets_1: Vec<Vec<OffsetSize>>,
        original_offsets_2: Option<Vec<Vec<OffsetSize>>>,
        mask_1: Vec<Mask>,
        mask_2: Option<Vec<Mask>>,
    ) -> (
        Vec<i64>,
        Vec<i8>,
        Vec<i8>,
        Vec<Option<Offset>>,
        Vec<Vec<OffsetSize>>,
        Vec<Mask>,
    ) {
        match *self {
            Self::Bert(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::Roberta(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::XLMRoberta(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::Marian(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::T5(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::Albert(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
        }
    }
/// Interface method to build input with special tokens
Interface method to build input with special tokens
Interface method to build input with special tokens
pub fn build_input_with_special_tokens(
        &self,
        tokens_1: Vec<i64>,
        tokens_2: Option<Vec<i64>>,
        offsets_1: Vec<Option<Offset>>,
        offsets_2: Option<Vec<Option<Offset>>>,
        original_offsets_1: Vec<Vec<OffsetSize>>,
        original_offsets_2: Option<Vec<Vec<OffsetSize>>>,
        mask_1: Vec<Mask>,
        mask_2: Option<Vec<Mask>>,
    ) -> (
        Vec<i64>,
        Vec<i8>,
        Vec<i8>,
        Vec<Option<Offset>>,
        Vec<Vec<OffsetSize>>,
        Vec<Mask>,
    ) {
        match *self {
            Self::Bert(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::Roberta(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::XLMRoberta(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::Marian(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::T5(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
            Self::Albert(ref tokenizer) => tokenizer.build_input_with_special_tokens(
                tokens_1,
                tokens_2,
                offsets_1,
                offsets_2,
                original_offsets_1,
                original_offsets_2,
                mask_1,
                mask_2,
            ),
        }
    }
pub fn build_input_with_special_tokens ( & self , tokens_1 : Vec < i64 > , tokens_2 : Option < Vec < i64 > > , offsets_1 : Vec < Option < Offset > > , offsets_2 : Option < Vec < Option < Offset > > > , original_offsets_1 : Vec < Vec < OffsetSize > > , original_offsets_2 : Option < Vec < Vec < OffsetSize > > > , mask_1 : Vec < Mask > , mask_2 : Option < Vec < Mask > > , ) -> ( Vec < i64 > , Vec < i8 > , Vec < i8 > , Vec < Option < Offset > > , Vec < Vec < OffsetSize > > , Vec < Mask > , ) { match * self { Self :: Bert ( ref tokenizer ) => tokenizer . build_input_with_special_tokens ( tokens_1 , tokens_2 , offsets_1 , offsets_2 , original_offsets_1 , original_offsets_2 , mask_1 , mask_2 , ) , Self :: Roberta ( ref tokenizer ) => tokenizer . build_input_with_special_tokens ( tokens_1 , tokens_2 , offsets_1 , offsets_2 , original_offsets_1 , original_offsets_2 , mask_1 , mask_2 , ) , Self :: XLMRoberta ( ref tokenizer ) => tokenizer . build_input_with_special_tokens ( tokens_1 , tokens_2 , offsets_1 , offsets_2 , original_offsets_1 , original_offsets_2 , mask_1 , mask_2 , ) , Self :: Marian ( ref tokenizer ) => tokenizer . build_input_with_special_tokens ( tokens_1 , tokens_2 , offsets_1 , offsets_2 , original_offsets_1 , original_offsets_2 , mask_1 , mask_2 , ) , Self :: T5 ( ref tokenizer ) => tokenizer . build_input_with_special_tokens ( tokens_1 , tokens_2 , offsets_1 , offsets_2 , original_offsets_1 , original_offsets_2 , mask_1 , mask_2 , ) , Self :: Albert ( ref tokenizer ) => tokenizer . build_input_with_special_tokens ( tokens_1 , tokens_2 , offsets_1 , offsets_2 , original_offsets_1 , original_offsets_2 , mask_1 , mask_2 , ) , } }
Interface method to build input with special tokens
Interface method to build input with special tokens
&self
tokens_1
Vec<i64>
tokens_2
Option<Vec<i64>>
offsets_1
Vec<Option<Offset>>
offsets_2
Option<Vec<Option<Offset>>>
original_offsets_1
Vec<Vec<OffsetSize>>
original_offsets_2
Option<Vec<Vec<OffsetSize>>>
mask_1
Vec<Mask>
mask_2
Option<Vec<Mask>>
&self
tokens_1
Vec<i64>
tokens_2
Option<Vec<i64>>
offsets_1
Vec<Option<Offset>>
offsets_2
Option<Vec<Option<Offset>>>
original_offsets_1
Vec<Vec<OffsetSize>>
original_offsets_2
Option<Vec<Vec<OffsetSize>>>
mask_1
Vec<Mask>
mask_2
Option<Vec<Mask>>