





from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFTransfoXLForSequenceClassification,
				    TFTransfoXLLMHeadModel,
				    TFTransfoXLModel,
				)

class        __SCREAMING_SNAKE_CASE					:






							def __init__(						self  , __lowerCAmelCase  , ):
														UpperCamelCase__				     =			parent
														UpperCamelCase__				     =			13
														UpperCamelCase__				     =			7
														UpperCamelCase__				     =			30
														UpperCamelCase__				     =			self.seq_length + self.mem_len
														UpperCamelCase__				     =			15
														UpperCamelCase__				     =			True
														UpperCamelCase__				     =			True
														UpperCamelCase__				     =			99
														UpperCamelCase__				     =			[10, 50, 80]
														UpperCamelCase__				     =			32
														UpperCamelCase__				     =			32
														UpperCamelCase__				     =			4
														UpperCamelCase__				     =			8
														UpperCamelCase__				     =			128
														UpperCamelCase__				     =			2
														UpperCamelCase__				     =			2
														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			1
														UpperCamelCase__				     =			0
														UpperCamelCase__				     =			3
														UpperCamelCase__				     =			self.vocab_size - 1
														UpperCamelCase__				     =			0.01






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size  )
														UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size  )

														UpperCamelCase__				     =			None
														if self.use_labels:
																					UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size  )

														UpperCamelCase__				     =			TransfoXLConfig(
														    vocab_size=self.vocab_size  , mem_len=self.mem_len  , clamp_len=self.clamp_len  , cutoffs=self.cutoffs  , d_model=self.hidden_size  , d_embed=self.d_embed  , n_head=self.num_attention_heads  , d_head=self.d_head  , d_inner=self.d_inner  , div_val=self.div_val  , n_layer=self.num_hidden_layers  , eos_token_id=self.eos_token_id  , pad_token_id=self.vocab_size - 1  , init_range=self.init_range  , num_labels=self.num_labels  , )

														return (config, input_ids_a, input_ids_a, lm_labels)






							def 				_lowerCamelCase		(						self  ):
														random.seed(self.seed  )
														tf.random.set_seed(self.seed  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			TFTransfoXLModel(__lowerCAmelCase  )

														UpperCamelCase__     ,     UpperCamelCase__				     =			model(__lowerCAmelCase  ).to_tuple()

														UpperCamelCase__				     =			{"""input_ids""": input_ids_a, """mems""": mems_a}

														UpperCamelCase__     ,     UpperCamelCase__				     =			model(__lowerCAmelCase  ).to_tuple()

														self.parent.assertEqual(hidden_states_a.shape  , (self.batch_size, self.seq_length, self.hidden_size)  )
														self.parent.assertEqual(hidden_states_a.shape  , (self.batch_size, self.seq_length, self.hidden_size)  )
														self.parent.assertListEqual(
														    [mem.shape for mem in mems_a]  , [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  , )
														self.parent.assertListEqual(
														    [mem.shape for mem in mems_a]  , [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  , )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			TFTransfoXLLMHeadModel(__lowerCAmelCase  )

														UpperCamelCase__     ,     UpperCamelCase__				     =			model(__lowerCAmelCase  ).to_tuple()

														UpperCamelCase__				     =			{"""input_ids""": input_ids_a, """labels""": lm_labels}
														UpperCamelCase__     ,     UpperCamelCase__				     =			model(__lowerCAmelCase  ).to_tuple()

														UpperCamelCase__     ,     UpperCamelCase__				     =			model([input_ids_a, mems_a]  ).to_tuple()

														UpperCamelCase__				     =			{"""input_ids""": input_ids_a, """mems""": mems_a, """labels""": lm_labels}

														UpperCamelCase__     ,     UpperCamelCase__				     =			model(__lowerCAmelCase  ).to_tuple()

														self.parent.assertEqual(lm_logits_a.shape  , (self.batch_size, self.seq_length, self.vocab_size)  )
														self.parent.assertListEqual(
														    [mem.shape for mem in mems_a]  , [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  , )

														self.parent.assertEqual(lm_logits_a.shape  , (self.batch_size, self.seq_length, self.vocab_size)  )
														self.parent.assertListEqual(
														    [mem.shape for mem in mems_a]  , [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  , )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			TFTransfoXLForSequenceClassification(__lowerCAmelCase  )
														UpperCamelCase__				     =			model(__lowerCAmelCase  )
														self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.num_labels)  )







							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.prepare_config_and_inputs()
														((UpperCamelCase__)     ,     (UpperCamelCase__)     ,     (UpperCamelCase__)     ,     (UpperCamelCase__))				     =			config_and_inputs
														UpperCamelCase__				     =			{"""input_ids""": input_ids_a}
														return config, inputs_dict






@require_tf
class        __SCREAMING_SNAKE_CASE					(     _a   ,		_a   ,		unittest.TestCase       ):
							snake_case     :      int						     =       (
							    (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
							)
							snake_case     :      Tuple						     =       () if is_tf_available() else ()
							snake_case     :      List[str]						     =       (
							    {
							        'feature-extraction': TFTransfoXLModel,
							        'text-classification': TFTransfoXLForSequenceClassification,
							        'text-generation': TFTransfoXLLMHeadModel,
							        'zero-shot': TFTransfoXLForSequenceClassification,
							    }
							    if is_tf_available()
							    else {}
							)
							# TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
							snake_case     :      Optional[int]						     =       False
							snake_case     :      Union[str, Any]						     =       False
							snake_case     :      Tuple						     =       False
							snake_case     :      Dict						     =       False






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):







														if pipeline_test_casse_name == "TextGenerationPipelineTests":
																					# Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
																					# `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
																					# tokenizer.
																					return True

														return False






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			TFTransfoXLModelTester(self  )
														UpperCamelCase__				     =			ConfigTester(self  , config_class=__lowerCAmelCase  , d_embed=37  )






							def 				_lowerCamelCase		(						self  ):
														self.config_tester.run_common_tests()






							def 				_lowerCamelCase		(						self  ):
														self.model_tester.set_seed()
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_transfo_xl_model(*__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														self.model_tester.set_seed()
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_transfo_xl_lm_head(*__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__     ,     UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs_for_common()
														UpperCamelCase__				     =			[TFTransfoXLForSequenceClassification]

														for model_class in self.all_model_classes:
																					UpperCamelCase__				     =			model_class(__lowerCAmelCase  )
																					assert isinstance(model.get_input_embeddings()  , tf.keras.layers.Layer  )
																					if model_class in list_other_models_with_output_ebd:
																												UpperCamelCase__				     =			model.get_output_embeddings()
																												assert isinstance(__lowerCAmelCase  , tf.keras.layers.Layer  )
																												UpperCamelCase__				     =			model.get_bias()
																												assert name is None
																					else:
																												UpperCamelCase__				     =			model.get_output_embeddings()
																												assert x is None
																												UpperCamelCase__				     =			model.get_bias()
																												assert name is None






							def 				_lowerCamelCase		(						self  ):
														pass






							@slow
							def 				_lowerCamelCase		(						self  ):







														for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					UpperCamelCase__				     =			TFTransfoXLModel.from_pretrained(__lowerCAmelCase  )
																					self.assertIsNotNone(__lowerCAmelCase  )







							@unittest.skip(reason="""This model doesn\'t play well with fit() due to not returning a single loss."""  )
							def 				_lowerCamelCase		(						self  ):
														pass






@require_tf
class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):






							@unittest.skip("""Skip test until #12651 is resolved."""  )
							@slow
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			TFTransfoXLLMHeadModel.from_pretrained("""transfo-xl-wt103"""  )
														# fmt: off
														UpperCamelCase__				     =			tf.convert_to_tensor([[33,1297,2,1,1009,4,1109,11739,4762,358,5,25,245,22,1706,17,20098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,6224,831,16002,2,8,603,78967,29546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,29546,54,8,3609,5,57211,49,4,1,277,18,8,1755,15691,3,341,25,416,693,42573,71,17,401,94,31,17919,2,29546,7873,18,1,435,23,11011,755,5,5167,3,7983,98,84,2,29546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,29546,824,1400,1868,2,19,160,2,311,8,5496,2,20920,17,25,15097,3,24,24,0]]  , dtype=tf.intaa  )  # noqa: E231
														# fmt: on
														#  In 1991 , the remains of Russian Tsar Nicholas II and his family
														#  ( except for Alexei and Maria ) are discovered .
														#  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
														#  remainder of the story . 1883 Western Siberia ,
														#  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
														#  Rasputin has a vision and denounces one of the men as a horse thief . Although his
														#  father initially slaps him for making such an accusation , Rasputin watches as the
														#  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
														#  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
														#  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

														# fmt: off
														UpperCamelCase__				     =			[33,1297,2,1,1009,4,1109,11739,4762,358,5,25,245,22,1706,17,20098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,6224,831,16002,2,8,603,78967,29546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,29546,54,8,3609,5,57211,49,4,1,277,18,8,1755,15691,3,341,25,416,693,42573,71,17,401,94,31,17919,2,29546,7873,18,1,435,23,11011,755,5,5167,3,7983,98,84,2,29546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,29546,824,1400,1868,2,19,160,2,311,8,5496,2,20920,17,25,15097,3,24,24,0,33,1,1857,2,1,1009,4,1109,11739,4762,358,5,25,245,28,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,0]  # noqa: E231
														# fmt: on
														#  In 1991, the remains of Russian Tsar Nicholas II and his family (
														#  except for Alexei and Maria ) are discovered. The voice of young son,
														#  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
														#  1883 Western Siberia, a young Grigori Rasputin is asked by his father
														#  and a group of men to perform magic. Rasputin has a vision and
														#  denounces one of the men as a horse thief. Although his father initially
														#  slaps him for making such an accusation, Rasputin watches as the man
														#  is chased outside and beaten. Twenty years later, Rasputin sees a vision
														#  of the Virgin Mary, prompting him to become a priest.
														#  Rasputin quickly becomes famous, with people, even a bishop, begging for
														#  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
														# Nicholas II and his family were discovered. The voice of <unk> young son,
														# Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

														UpperCamelCase__				     =			model.generate(__lowerCAmelCase  , max_length=200  , do_sample=__lowerCAmelCase  )
														self.assertListEqual(output_ids[0].numpy().tolist()  , __lowerCAmelCase  )











import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    """files"""   ,			[
        ["""full:README.md""", """dataset_infos.json"""],
        ["""empty:README.md""", """dataset_infos.json"""],
        ["""dataset_infos.json"""],
        ["""full:README.md"""],
    ]   ,			)
def     _UpperCamelCase     (a__   :Any   ,			a__   :Union[str, Any]       ):






							"""simple docstring"""





							UpperCamelCase__				     =			tmp_path_factory.mktemp("""dset_infos_dir"""       )
							if "full:README.md" in files:
														with open(dataset_infos_dir / """README.md"""   ,			"""w"""       ) as f:
																					f.write("""---\ndataset_info:\n  dataset_size: 42\n---"""       )
							if "empty:README.md" in files:
														with open(dataset_infos_dir / """README.md"""   ,			"""w"""       ) as f:
																					f.write(""""""       )
    # we want to support dataset_infos.json for backward compatibility
							if "dataset_infos.json" in files:
														with open(dataset_infos_dir / """dataset_infos.json"""   ,			"""w"""       ) as f:
																					f.write("""{\"default\": {\"dataset_size\": 42}}"""       )
							UpperCamelCase__				     =			DatasetInfosDict.from_directory(a__       )
							assert dataset_infos
							assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    """dataset_info"""   ,			[
        DatasetInfo(),
        DatasetInfo(
            description="""foo"""   ,			features=Features({"""a""": Value("""int32"""       )}       )   ,			builder_name="""builder"""   ,			config_name="""config"""   ,			version="""1.0.0"""   ,			splits=[{"""name""": """train"""}]   ,			download_size=42   ,			),
    ]   ,			)
def     _UpperCamelCase     (a__   :Optional[int]   ,			a__   :DatasetInfo       ):






							"""simple docstring"""





							UpperCamelCase__				     =			str(a__       )
							dataset_info.write_to_directory(a__       )
							UpperCamelCase__				     =			DatasetInfo.from_directory(a__       )
							assert dataset_info == reloaded
							assert os.path.exists(os.path.join(a__   ,			"""dataset_info.json"""       )       )





def     _UpperCamelCase     ():






							"""simple docstring"""





							UpperCamelCase__				     =			DatasetInfo(
							    description="""foo"""   ,			citation="""bar"""   ,			homepage="""https://foo.bar"""   ,			license="""CC0"""   ,			features=Features({"""a""": Value("""int32"""       )}       )   ,			post_processed={}   ,			supervised_keys=()   ,			task_templates=[]   ,			builder_name="""builder"""   ,			config_name="""config"""   ,			version="""1.0.0"""   ,			splits=[{"""name""": """train""", """num_examples""": 42}]   ,			download_checksums={}   ,			download_size=1337   ,			post_processing_size=442   ,			dataset_size=1234   ,			size_in_bytes=1337 + 442 + 1234   ,			)
							UpperCamelCase__				     =			dataset_info._to_yaml_dict()
							assert sorted(a__       ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML       )
							for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
														assert key in dataset_info_yaml_dict
														assert isinstance(dataset_info_yaml_dict[key]   ,			(list, dict, int, str)       )
							UpperCamelCase__				     =			yaml.safe_dump(a__       )
							UpperCamelCase__				     =			yaml.safe_load(a__       )
							assert dataset_info_yaml_dict == reloaded





def     _UpperCamelCase     ():






							"""simple docstring"""





							UpperCamelCase__				     =			DatasetInfo()
							UpperCamelCase__				     =			dataset_info._to_yaml_dict()
							assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    """dataset_infos_dict"""   ,			[
        DatasetInfosDict(),
        DatasetInfosDict({"""default""": DatasetInfo()}       ),
        DatasetInfosDict({"""my_config_name""": DatasetInfo()}       ),
        DatasetInfosDict(
            {
                """default""": DatasetInfo(
                    description="""foo"""   ,			features=Features({"""a""": Value("""int32"""       )}       )   ,			builder_name="""builder"""   ,			config_name="""config"""   ,			version="""1.0.0"""   ,			splits=[{"""name""": """train"""}]   ,			download_size=42   ,			)
            }       ),
        DatasetInfosDict(
            {
                """v1""": DatasetInfo(dataset_size=42       ),
                """v2""": DatasetInfo(dataset_size=1337       ),
            }       ),
    ]   ,			)
def     _UpperCamelCase     (a__   :int   ,			a__   :DatasetInfosDict       ):






							"""simple docstring"""





							UpperCamelCase__				     =			str(a__       )
							dataset_infos_dict.write_to_directory(a__       )
							UpperCamelCase__				     =			DatasetInfosDict.from_directory(a__       )

							# the config_name of the dataset_infos_dict take over the attribute
							for config_name, dataset_info in dataset_infos_dict.items():
														UpperCamelCase__				     =			config_name
														# the yaml representation doesn't include fields like description or citation
														# so we just test that we can recover what we can from the yaml
														UpperCamelCase__				     =			DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()       )
							assert dataset_infos_dict == reloaded

							if dataset_infos_dict:
														assert os.path.exists(os.path.join(a__   ,			"""README.md"""       )       )




