



"""simple docstring"""






import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	List[Any]           =  PhobertTokenizer
   A__		:	Optional[Any]           =  False



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     _snake_case   	=      ['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
     _snake_case   	=      dict(zip(__lowerCamelCase							,       range(len(__lowerCamelCase				)				)				)				)
     _snake_case   	=      ['''#version: 0.2''', '''l à</w>''']
     _snake_case   	=      {'''unk_token''': '''<unk>'''}

     _snake_case   	=      os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['''vocab_file''']				)
     _snake_case   	=      os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['''merges_file''']				)

     with open(self.vocab_file							,       '''w'''							,       encoding='''utf-8'''				) as fp:
       for token in vocab_tokens:
         fp.write(f"""{token} {vocab_tokens[token]}\n"""				)
     with open(self.merges_file							,       '''w'''							,       encoding='''utf-8'''				) as fp:
       fp.write('''\n'''.join(__lowerCamelCase				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       **__lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     kwargs.update(self.special_tokens_map				)
     return PhobertTokenizer.from_pretrained(self.tmpdirname							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      '''Tôi là VinAI Research'''
     _snake_case   	=      '''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
     return input_text, output_text





   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      PhobertTokenizer(self.vocab_file							,       self.merges_file							,       **self.special_tokens_map				)
     _snake_case   	=      '''Tôi là VinAI Research'''
     _snake_case   	=      '''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
     _snake_case   	=      tokenizer.tokenize(__lowerCamelCase				)
     print(__lowerCamelCase				)
     self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)

     _snake_case   	=      tokens + [tokenizer.unk_token]

     _snake_case   	=      [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase				)							,       __lowerCamelCase				)












import numpy as np





def 				__lowerCAmelCase(       _SCREAMING_SNAKE_CASE				)							-> np.array:




    """simple docstring"""




    return (2 / (1 + np.exp(-2 * vector				))) - 1


if __name__ == "__main__":
 import doctest

 doctest.testmod()





