
"""simple docstring"""




import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class 			__magic_name__       (       SCREAMING_SNAKE_CASE__   ,					unittest.TestCase ):
       UpperCamelCase_   =		PhobertTokenizer
       UpperCamelCase_   =		False
       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         _lowercase:			List[str]       =     ['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
         _lowercase:			Dict       =     dict(zip(A_ ,				range(len(A_       )       )       )       )
         _lowercase:			Tuple       =     ['''#version: 0.2''', '''l à</w>''']
         _lowercase:			Optional[int]       =     {'''unk_token''': '''<unk>'''}

         _lowercase:			List[Any]       =     os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES['''vocab_file''']       )
         _lowercase:			Union[str, Any]       =     os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES['''merges_file''']       )

         with open(self.vocab_file ,				'''w''' ,				encoding='''utf-8'''       ) as fp:
           for token in vocab_tokens:
             fp.write(f'''{token} {vocab_tokens[token]}\n'''       )
         with open(self.merges_file ,				'''w''' ,				encoding='''utf-8'''       ) as fp:
           fp.write('''\n'''.join(A_       )       )
       def 							lowercase_							(     self ,				**A_       )     ->	Tuple:





         """simple docstring"""




         kwargs.update(self.special_tokens_map       )
         return PhobertTokenizer.from_pretrained(self.tmpdirname ,				**A_       )
       def 							lowercase_							(     self ,				A_       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			Dict       =     '''Tôi là VinAI Research'''
         _lowercase:			List[str]       =     '''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
         return input_text, output_text



       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			str       =     PhobertTokenizer(self.vocab_file ,				self.merges_file ,				**self.special_tokens_map       )
         _lowercase:			Dict       =     '''Tôi là VinAI Research'''
         _lowercase:			str       =     '''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
         _lowercase:			List[str]       =     tokenizer.tokenize(A_       )
         print(A_       )
         self.assertListEqual(A_ ,				A_       )

         _lowercase:			Optional[int]       =     tokens + [tokenizer.unk_token]

         _lowercase:			List[str]       =     [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A_       ) ,				A_       )


"""simple docstring"""




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  if not isinstance(_UpperCamelCase ,				_UpperCamelCase						):
    raise ValueError('''Input series is not valid, valid series - [2, 4, 6]'''						)
  if len(_UpperCamelCase						) == 0:
    raise ValueError('''Input list must be a non empty list'''						)
  if len(_UpperCamelCase						) == 1:
    return True
  _lowercase:			Any       =     series[1] - series[0]
  for index in range(len(_UpperCamelCase						) - 1						):
    if series[index + 1] - series[index] != common_diff:
      return False
  return True




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  if not isinstance(_UpperCamelCase ,				_UpperCamelCase						):
    raise ValueError('''Input series is not valid, valid series - [2, 4, 6]'''						)
  if len(_UpperCamelCase						) == 0:
    raise ValueError('''Input list must be a non empty list'''						)
  _lowercase:			Tuple       =     0
  for val in series:
    answer += val
  return answer / len(_UpperCamelCase						)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
