pub fn convert_tokens_to_ids(&self, tokens: &Vec<String>) -> Vec<i64> {
        match *self {
            Self::Bert(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::Roberta(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::Marian(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::T5(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::XLMRoberta(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::Albert(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
        }
    }
/// Interface method to convert tokens to ids
Interface method to convert tokens to ids
Interface method to convert tokens to ids
pub fn convert_tokens_to_ids(&self, tokens: &Vec<String>) -> Vec<i64> {
        match *self {
            Self::Bert(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::Roberta(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::Marian(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::T5(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::XLMRoberta(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
            Self::Albert(ref tokenizer) => tokenizer.convert_tokens_to_ids(tokens),
        }
    }
pub fn convert_tokens_to_ids ( & self , tokens : & Vec < String > ) -> Vec < i64 > { match * self { Self :: Bert ( ref tokenizer ) => tokenizer . convert_tokens_to_ids ( tokens ) , Self :: Roberta ( ref tokenizer ) => tokenizer . convert_tokens_to_ids ( tokens ) , Self :: Marian ( ref tokenizer ) => tokenizer . convert_tokens_to_ids ( tokens ) , Self :: T5 ( ref tokenizer ) => tokenizer . convert_tokens_to_ids ( tokens ) , Self :: XLMRoberta ( ref tokenizer ) => tokenizer . convert_tokens_to_ids ( tokens ) , Self :: Albert ( ref tokenizer ) => tokenizer . convert_tokens_to_ids ( tokens ) , } }
Interface method to convert tokens to ids
Interface method to convert tokens to ids
&self
tokens
&Vec<String>
&self
tokens
&Vec<String>