






from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
	from transformers import (
	    AutoConfig,
	    BertConfig,
	    GPTaConfig,
	    TaConfig,
	    TFAutoModel,
	    TFAutoModelForCausalLM,
	    TFAutoModelForMaskedLM,
	    TFAutoModelForPreTraining,
	    TFAutoModelForQuestionAnswering,
	    TFAutoModelForSeqaSeqLM,
	    TFAutoModelForSequenceClassification,
	    TFAutoModelWithLMHead,
	    TFBertForMaskedLM,
	    TFBertForPreTraining,
	    TFBertForQuestionAnswering,
	    TFBertForSequenceClassification,
	    TFBertModel,
	    TFGPTaLMHeadModel,
	    TFRobertaForMaskedLM,
	    TFTaForConditionalGeneration,
	)
	from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
	from transformers import (
	    AutoModel,
	    AutoModelForCausalLM,
	    AutoModelForMaskedLM,
	    AutoModelForPreTraining,
	    AutoModelForQuestionAnswering,
	    AutoModelForSeqaSeqLM,
	    AutoModelForSequenceClassification,
	    AutoModelWithLMHead,
	    BertForMaskedLM,
	    BertForPreTraining,
	    BertForQuestionAnswering,
	    BertForSequenceClassification,
	    BertModel,
	    GPTaLMHeadModel,
	    RobertaForMaskedLM,
	    TaForConditionalGeneration,
	)



@is_pt_tf_cross_test
class 							SCREAMING_SNAKE_CASE__						(    unittest.TestCase						):





					"""simple docstring"""






					@slow
					def 	_lowercase					( self					:  Optional[int]	):
						# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						for model_name in ["bert-base-uncased"]:
							snake_case__			:				Union[str, Any]       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Any       =  TFAutoModel.from_pretrained(__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				List[Any]       =  AutoModel.from_pretrained(__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  Optional[int]	):
						# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						for model_name in ["bert-base-uncased"]:
							snake_case__			:				Union[str, Any]       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				List[Any]       =  TFAutoModelForPreTraining.from_pretrained(__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Any       =  AutoModelForPreTraining.from_pretrained(__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  Optional[int]	):
						for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							snake_case__			:				Tuple       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				List[Any]       =  TFAutoModelForCausalLM.from_pretrained(__A							,		from_pt=__A	)
							snake_case__,						snake_case__			:				Optional[int]       =  TFAutoModelForCausalLM.from_pretrained(
							    __A							,		output_loading_info=__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Any       =  AutoModelForCausalLM.from_pretrained(__A							,		from_tf=__A	)
							snake_case__,						snake_case__			:				Optional[Any]       =  AutoModelForCausalLM.from_pretrained(
							    __A							,		output_loading_info=__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  str	):
						for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							snake_case__			:				Optional[int]       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				List[Any]       =  TFAutoModelWithLMHead.from_pretrained(__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Any       =  AutoModelWithLMHead.from_pretrained(__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  Dict	):
						for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							snake_case__			:				Union[str, Any]       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Union[str, Any]       =  TFAutoModelForMaskedLM.from_pretrained(__A							,		from_pt=__A	)
							snake_case__,						snake_case__			:				Tuple       =  TFAutoModelForMaskedLM.from_pretrained(
							    __A							,		output_loading_info=__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				int       =  AutoModelForMaskedLM.from_pretrained(__A							,		from_tf=__A	)
							snake_case__,						snake_case__			:				List[Any]       =  AutoModelForMaskedLM.from_pretrained(
							    __A							,		output_loading_info=__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  Union[str, Any]	):
						for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							snake_case__			:				Optional[Any]       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Optional[int]       =  TFAutoModelForSeqaSeqLM.from_pretrained(__A							,		from_pt=__A	)
							snake_case__,						snake_case__			:				Tuple       =  TFAutoModelForSeqaSeqLM.from_pretrained(
							    __A							,		output_loading_info=__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Any       =  AutoModelForSeqaSeqLM.from_pretrained(__A							,		from_tf=__A	)
							snake_case__,						snake_case__			:				Union[str, Any]       =  AutoModelForSeqaSeqLM.from_pretrained(
							    __A							,		output_loading_info=__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  Optional[int]	):
						# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						for model_name in ["bert-base-uncased"]:
							snake_case__			:				int       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Optional[Any]       =  TFAutoModelForSequenceClassification.from_pretrained(__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Any       =  AutoModelForSequenceClassification.from_pretrained(__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					@slow
					def 	_lowercase					( self					:  Optional[int]	):
						# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						for model_name in ["bert-base-uncased"]:
							snake_case__			:				List[Any]       =  AutoConfig.from_pretrained(__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				Tuple       =  TFAutoModelForQuestionAnswering.from_pretrained(__A							,		from_pt=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)

							snake_case__			:				List[Any]       =  AutoModelForQuestionAnswering.from_pretrained(__A							,		from_tf=__A	)
							self.assertIsNotNone(__A	)
							self.assertIsInstance(__A							,		__A	)



					def 	_lowercase					( self					:  str	):
						snake_case__			:				List[Any]       =  TFAutoModelWithLMHead.from_pretrained(__A							,		from_pt=__A	)
						self.assertIsInstance(__A							,		__A	)
						self.assertEqual(model.num_parameters()							,		1_4_4_1_0	)
						self.assertEqual(model.num_parameters(only_trainable=__A	)							,		1_4_4_1_0	)

						snake_case__			:				Tuple       =  AutoModelWithLMHead.from_pretrained(__A							,		from_tf=__A	)
						self.assertIsInstance(__A							,		__A	)
						self.assertEqual(model.num_parameters()							,		1_4_4_1_0	)
						self.assertEqual(model.num_parameters(only_trainable=__A	)							,		1_4_4_1_0	)






					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				Tuple       =  TFAutoModelWithLMHead.from_pretrained(__A							,		from_pt=__A	)
						self.assertIsInstance(__A							,		__A	)
						self.assertEqual(model.num_parameters()							,		1_4_4_1_0	)
						self.assertEqual(model.num_parameters(only_trainable=__A	)							,		1_4_4_1_0	)

						snake_case__			:				Dict       =  AutoModelWithLMHead.from_pretrained(__A							,		from_tf=__A	)
						self.assertIsInstance(__A							,		__A	)
						self.assertEqual(model.num_parameters()							,		1_4_4_1_0	)
						self.assertEqual(model.num_parameters(only_trainable=__A	)							,		1_4_4_1_0	)











import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		PhobertTokenizer
					a_     =		False



					def 	_lowercase					( self					:  List[str]	):
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						snake_case__			:				Optional[int]       =  ["T@@", "i", "I", "R@@", "r", "e@@"]
						snake_case__			:				int       =  dict(zip(__A							,		range(len(__A	)	)	)	)
						snake_case__			:				Dict       =  ["#version: 0.2", "l à</w>"]
						snake_case__			:				Optional[Any]       =  {"unk_token": "<unk>"}

						snake_case__			:				Tuple       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["vocab_file"]	)
						snake_case__			:				Union[str, Any]       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["merges_file"]	)

						with open(self.vocab_file							,		"w"							,		encoding="utf-8"	) as fp:
							for token in vocab_tokens:
								fp.write(f'''{token} {vocab_tokens[token]}\n'''	)
						with open(self.merges_file							,		"w"							,		encoding="utf-8"	) as fp:
							fp.write("\n".join(__A	)	)



					def 	_lowercase					( self					:  List[str]							,		**__A					:  Any	):
						kwargs.update(self.special_tokens_map	)
						return PhobertTokenizer.from_pretrained(self.tmpdirname							,		**__A	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  List[Any]	):
						snake_case__			:				str       =  "Tôi là VinAI Research"
						snake_case__			:				int       =  "T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
						return input_text, output_text






					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				int       =  PhobertTokenizer(self.vocab_file							,		self.merges_file							,		**self.special_tokens_map	)
						snake_case__			:				Tuple       =  "Tôi là VinAI Research"
						snake_case__			:				List[Any]       =  "T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
						snake_case__			:				int       =  tokenizer.tokenize(__A	)
						print(__A	)
						self.assertListEqual(__A							,		__A	)

						snake_case__			:				Any       =  tokens + [tokenizer.unk_token]

						snake_case__			:				Any       =  [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		__A	)



