pub fn predict(
        &self,
        input: &[&str],
        consolidate_sub_tokens: bool,
        return_special: bool,
    ) -> Vec<Token> {
        let (tokenized_input, input_tensor) = self.prepare_for_model(input.to_vec());
        let output = no_grad(|| {
            self.token_sequence_classifier.forward_t(
                Some(input_tensor.copy()),
                None,
                None,
                None,
                None,
                false,
            )
        });
        let output = output.detach().to(Device::Cpu);
        let score: Tensor = output.exp() / output.exp().sum1(&[-1], true, Float);
        let labels_idx = &score.argmax(-1, true);
        let mut tokens: Vec<Token> = vec![];
        for sentence_idx in 0..labels_idx.size()[0] {
            let labels = labels_idx.get(sentence_idx);
            let sentence_tokens = &tokenized_input[sentence_idx as usize];
            let original_chars = input[sentence_idx as usize].chars().collect_vec();
            let mut word_idx: u16 = 0;
            for position_idx in 0..sentence_tokens.token_ids.len() {
                let mask = sentence_tokens.mask[position_idx];
                if (mask == Mask::Special) & (!return_special) {
                    continue;
                }
                if !(mask == Mask::Continuation) {
                    word_idx += 1;
                }
                let token = {
                    self.decode_token(
                        &original_chars,
                        sentence_tokens,
                        &input_tensor,
                        &labels,
                        &score,
                        sentence_idx,
                        position_idx as i64,
                        word_idx - 1,
                    )
                };
                tokens.push(token);
            }
        }
        if consolidate_sub_tokens {
            self.consolidate_tokens(&mut tokens, &self.label_aggregation_function);
        }
        tokens
    }
/// Classify tokens in a text sequence
///
/// # Arguments
///
/// * `input` - `&[&str]` Array of texts to extract entities from.
/// * `consolidate_subtokens` - bool flag indicating if subtokens should be consolidated at the token level
/// * `return_special` - bool flag indicating if labels for special tokens should be returned
///
/// # Returns
///
/// * `Vec<Token>` containing Tokens with associated labels (for example POS tags)
///
/// # Example
///
/// ```no_run
/// # fn main() -> anyhow::Result<()> {
/// # use rust_bert::pipelines::token_classification::TokenClassificationModel;
///
/// let ner_model = TokenClassificationModel::new(Default::default())?;
/// let input = [
///     "My name is Amy. I live in Paris.",
///     "Paris is a city in France.",
/// ];
/// let output = ner_model.predict(&input, true, true);
/// # Ok(())
/// # }
/// ```
Classify tokens in a text sequence
Classify tokens in a text sequence
pub fn predict(
        &self,
        input: &[&str],
        consolidate_sub_tokens: bool,
        return_special: bool,
    ) -> Vec<Token> {
        let (tokenized_input, input_tensor) = self.prepare_for_model(input.to_vec());
        let output = no_grad(|| {
            self.token_sequence_classifier.forward_t(
                Some(input_tensor.copy()),
                None,
                None,
                None,
                None,
                false,
            )
        });
        let output = output.detach().to(Device::Cpu);
        let score: Tensor = output.exp() / output.exp().sum1(&[-1], true, Float);
        let labels_idx = &score.argmax(-1, true);
        let mut tokens: Vec<Token> = vec![];
        for sentence_idx in 0..labels_idx.size()[0] {
            let labels = labels_idx.get(sentence_idx);
            let sentence_tokens = &tokenized_input[sentence_idx as usize];
            let original_chars = input[sentence_idx as usize].chars().collect_vec();
            let mut word_idx: u16 = 0;
            for position_idx in 0..sentence_tokens.token_ids.len() {
                let mask = sentence_tokens.mask[position_idx];
                if (mask == Mask::Special) & (!return_special) {
                    continue;
                }
                if !(mask == Mask::Continuation) {
                    word_idx += 1;
                }
                let token = {
                    self.decode_token(
                        &original_chars,
                        sentence_tokens,
                        &input_tensor,
                        &labels,
                        &score,
                        sentence_idx,
                        position_idx as i64,
                        word_idx - 1,
                    )
                };
                tokens.push(token);
            }
        }
        if consolidate_sub_tokens {
            self.consolidate_tokens(&mut tokens, &self.label_aggregation_function);
        }
        tokens
    }
pub fn predict ( & self , input : & [ & str ] , consolidate_sub_tokens : bool , return_special : bool , ) -> Vec < Token > { let ( tokenized_input , input_tensor ) = self . prepare_for_model ( input . to_vec ( ) ) ; let output = no_grad ( | | { self . token_sequence_classifier . forward_t ( Some ( input_tensor . copy ( ) ) , None , None , None , None , false , ) } ) ; let output = output . detach ( ) . to ( Device :: Cpu ) ; let score : Tensor = output . exp ( ) / output . exp ( ) . sum1 ( & [ - 1 ] , true , Float ) ; let labels_idx = & score . argmax ( - 1 , true ) ; let mut tokens : Vec < Token > = vec ! [ ] ; for sentence_idx in 0 .. labels_idx . size ( ) [ 0 ] { let labels = labels_idx . get ( sentence_idx ) ; let sentence_tokens = & tokenized_input [ sentence_idx as usize ] ; let original_chars = input [ sentence_idx as usize ] . chars ( ) . collect_vec ( ) ; let mut word_idx : u16 = 0 ; for position_idx in 0 .. sentence_tokens . token_ids . len ( ) { let mask = sentence_tokens . mask [ position_idx ] ; if ( mask == Mask :: Special ) & ( ! return_special ) { continue ; } if ! ( mask == Mask :: Continuation ) { word_idx += 1 ; } let token = { self . decode_token ( & original_chars , sentence_tokens , & input_tensor , & labels , & score , sentence_idx , position_idx as i64 , word_idx - 1 , ) } ; tokens . push ( token ) ; } } if consolidate_sub_tokens { self . consolidate_tokens ( & mut tokens , & self . label_aggregation_function ) ; } tokens }
Classify tokens in a text sequence
Classify tokens in a text sequence
&self
input
&[&str]
consolidate_sub_tokens
bool
return_special
bool
`Vec` containing Tokens with associated labels (for example POS tags)
` Vec ` containing Tokens with associated labels ( for example POS tags )
&self
input
&[&str]
consolidate_sub_tokens
bool
return_special
bool
example
None
fn main() -> anyhow::result<()> {
None
use rust_bert::pipelines::token_classification::tokenclassificationmodel;
let ner_model = TokenClassificationModel::new(Default::default())?;
let input = [
"My name is Amy. I live in Paris.",
"Paris is a city in France.",
];
let output = ner_model.predict(&input, true, true).
let ner_model = TokenClassificationModel :: new ( Default :: default () ) ? ; let input = [ " My name is Amy . I live in Paris . " " Paris is a city in France . " ] ; let output = ner_model . predict ( &input true true ) .
ok(())
None
}
None