torchfly/text/bidaf/bidaf_pipeline.py
def process_one(self, text: str)->Dict[str, Any]:
        "Returns word tokens and character tokens"

        word_tokens = self.word_tokenizer(text)
        char_tokens = CharTokenizer.tokenize(word_tokens)

        word_tokens = [t.lower() for t in word_tokens]
        # numericalize
        word_tokens = np.array(self.word_indexer.numericalize(word_tokens))
        char_tokens = BiDAFCharIndexer.numericalize(char_tokens)

        return {"word_tokens": word_tokens,
                "char_tokens": char_tokens}
Returns word tokens and character tokens
Returns word tokens and character tokens
Returns word tokens and character tokens
def process_one(self, text: str)->Dict[str, Any]:
        word_tokens = self.word_tokenizer(text)
        char_tokens = CharTokenizer.tokenize(word_tokens)
        word_tokens = [t.lower() for t in word_tokens]
        word_tokens = np.array(self.word_indexer.numericalize(word_tokens))
        char_tokens = BiDAFCharIndexer.numericalize(char_tokens)
        return {"word_tokens": word_tokens,
                "char_tokens": char_tokens}
def process_one ( self , text : str ) -> Dict [ str , Any ] : word_tokens = self . word_tokenizer ( text ) char_tokens = CharTokenizer . tokenize ( word_tokens ) word_tokens = [ t . lower ( ) for t in word_tokens ] word_tokens = np . array ( self . word_indexer . numericalize ( word_tokens ) ) char_tokens = BiDAFCharIndexer . numericalize ( char_tokens ) return { "word_tokens" : word_tokens , "char_tokens" : char_tokens }
Returns word tokens and character tokens
Returns word tokens and character tokens
"Returns word tokens and character tokens" # numericalize
self
text
str