



'''simple docstring'''




import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class 		_UpperCamelCase						(							lowerCamelCase__					,      unittest.TestCase					):


			'''simple docstring'''

			_A   :	Any    					=	PhobertTokenizer
			_A   :	Optional[int]    					=	False

			def UpperCamelCase__      (							self  :				Union[str, Any]		):





					"""simple docstring"""






					super().setUp()

					# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
					__SCREAMING_SNAKE_CASE :	str					 =	["""T@@""", """i""", """I""", """R@@""", """r""", """e@@"""]
					__SCREAMING_SNAKE_CASE :	List[str]					 =	dict(zip(lowerCAmelCase__							,  range(len(lowerCAmelCase__		)		)		)		)
					__SCREAMING_SNAKE_CASE :	Any					 =	["""#version: 0.2""", """l à</w>"""]
					__SCREAMING_SNAKE_CASE :	Any					 =	{"""unk_token""": """<unk>"""}

					__SCREAMING_SNAKE_CASE :	Tuple					 =	os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES["""vocab_file"""]		)
					__SCREAMING_SNAKE_CASE :	Any					 =	os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES["""merges_file"""]		)

					with open(self.vocab_file							,  """w"""							,  encoding="""utf-8"""		) as fp:
							for token in vocab_tokens:
									fp.write(F"{token} {vocab_tokens[token]}\n"		)
					with open(self.merges_file							,  """w"""							,  encoding="""utf-8"""		) as fp:
							fp.write("""\n""".join(lowerCAmelCase__		)		)

			def UpperCamelCase__      (							self  :				Tuple							,  **lowerCAmelCase__  :				Any		):





					"""simple docstring"""






					kwargs.update(self.special_tokens_map		)
					return PhobertTokenizer.from_pretrained(self.tmpdirname							,  **lowerCAmelCase__		)

			def UpperCamelCase__      (							self  :				Optional[int]							,  lowerCAmelCase__  :				str		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	List[Any]					 =	"""Tôi là VinAI Research"""
					__SCREAMING_SNAKE_CASE :	Tuple					 =	"""T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
					return input_text, output_text



			def UpperCamelCase__      (							self  :				List[str]		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	PhobertTokenizer(self.vocab_file							,  self.merges_file							,  **self.special_tokens_map		)
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	"""Tôi là VinAI Research"""
					__SCREAMING_SNAKE_CASE :	int					 =	"""T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	tokenizer.tokenize(lowerCAmelCase__		)
					print(lowerCAmelCase__		)
					self.assertListEqual(lowerCAmelCase__							,  lowerCAmelCase__		)

					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	tokens + [tokenizer.unk_token]

					__SCREAMING_SNAKE_CASE :	Dict					 =	[4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__		)							,  lowerCAmelCase__		)




'''simple docstring'''




from ..utils import DummyObject, requires_backends






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Optional[int]    					=	['''onnx''']

			def __init__(							self  :				List[str]							,  *lowerCAmelCase__  :				Optional[int]							,  **lowerCAmelCase__  :				List[Any]		):





					"""simple docstring"""






					requires_backends(self							,  ["""onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				Optional[int]							,  *lowerCAmelCase__  :				int							,  **lowerCAmelCase__  :				int		):





					"""simple docstring"""






					requires_backends(cls							,  ["""onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				Tuple							,  *lowerCAmelCase__  :				List[str]							,  **lowerCAmelCase__  :				Optional[Any]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""onnx"""]		)