paddlenlp/transformers/layoutxlm/tokenizer.py
def _convert_token_to_id(self, token):
        """ Converts a token (str) in an id using the vocab. """
        if token in self.tokens_to_ids:
            return self.tokens_to_ids[token]
        spm_id = self.sp_model.PieceToId(token)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.offset if spm_id else self.unk_token_id
 Converts a token (str) in an id using the vocab. 
Converts a token (str) in an id using the vocab.
Converts a token ( str ) in an id using the vocab .
def _convert_token_to_id(self, token):
        if token in self.tokens_to_ids:
            return self.tokens_to_ids[token]
        spm_id = self.sp_model.PieceToId(token)
        return spm_id + self.offset if spm_id else self.unk_token_id
def _convert_token_to_id ( self , token ) : if token in self . tokens_to_ids : return self . tokens_to_ids [ token ] spm_id = self . sp_model . PieceToId ( token ) return spm_id + self . offset if spm_id else self . unk_token_id
Converts a token (str) in an id using the vocab.
Converts a token ( str ) in an id using the vocab .
""" Converts a token (str) in an id using the vocab. """ # Need to return unknown token if the SP model returned 0
self
token