




import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
       CTRLForSequenceClassification,
       CTRLLMHeadModel,
       CTRLModel,
   )





class 			lowerCamelCase_				:


 '''simple docstring'''







 def __init__(				self    :						Optional[Any]							,							_lowerCAmelCase    :						Union[str, Any]							,							_lowerCAmelCase    :						List[Any]=14							,							_lowerCAmelCase    :						int=7							,							_lowerCAmelCase    :						Union[str, Any]=True							,							_lowerCAmelCase    :						Dict=True							,							_lowerCAmelCase    :						Optional[int]=True							,							_lowerCAmelCase    :						Optional[Any]=True							,							_lowerCAmelCase    :						Any=True							,							_lowerCAmelCase    :						List[Any]=99							,							_lowerCAmelCase    :						Dict=32							,							_lowerCAmelCase    :						Optional[int]=5							,							_lowerCAmelCase    :						int=4							,							_lowerCAmelCase    :						Union[str, Any]=37							,							_lowerCAmelCase    :						List[str]="gelu"							,							_lowerCAmelCase    :						Any=0.1							,							_lowerCAmelCase    :						str=0.1							,							_lowerCAmelCase    :						Tuple=512							,							_lowerCAmelCase    :						Dict=16							,							_lowerCAmelCase    :						Any=2							,							_lowerCAmelCase    :						Tuple=0.02							,							_lowerCAmelCase    :						List[Any]=3							,							_lowerCAmelCase    :						Union[str, Any]=4							,							_lowerCAmelCase    :						List[str]=None							,							):
       SCREAMING_SNAKE_CASE_      		=     parent
       SCREAMING_SNAKE_CASE_      		=     batch_size
       SCREAMING_SNAKE_CASE_      		=     seq_length
       SCREAMING_SNAKE_CASE_      		=     is_training
       SCREAMING_SNAKE_CASE_      		=     use_token_type_ids
       SCREAMING_SNAKE_CASE_      		=     use_input_mask
       SCREAMING_SNAKE_CASE_      		=     use_labels
       SCREAMING_SNAKE_CASE_      		=     use_mc_token_ids
       SCREAMING_SNAKE_CASE_      		=     vocab_size
       SCREAMING_SNAKE_CASE_      		=     hidden_size
       SCREAMING_SNAKE_CASE_      		=     num_hidden_layers
       SCREAMING_SNAKE_CASE_      		=     num_attention_heads
       SCREAMING_SNAKE_CASE_      		=     intermediate_size
       SCREAMING_SNAKE_CASE_      		=     hidden_act
       SCREAMING_SNAKE_CASE_      		=     hidden_dropout_prob
       SCREAMING_SNAKE_CASE_      		=     attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE_      		=     max_position_embeddings
       SCREAMING_SNAKE_CASE_      		=     type_vocab_size
       SCREAMING_SNAKE_CASE_      		=     type_sequence_label_size
       SCREAMING_SNAKE_CASE_      		=     initializer_range
       SCREAMING_SNAKE_CASE_      		=     num_labels
       SCREAMING_SNAKE_CASE_      		=     num_choices
       SCREAMING_SNAKE_CASE_      		=     scope
       SCREAMING_SNAKE_CASE_      		=     self.vocab_size - 1



 def  lowerCAmelCase_    (				self    :						Any		):
       SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size		)

       SCREAMING_SNAKE_CASE_      		=     None
       if self.use_input_mask:
             SCREAMING_SNAKE_CASE_      		=     random_attention_mask([self.batch_size, self.seq_length]		)

       SCREAMING_SNAKE_CASE_      		=     None
       if self.use_token_type_ids:
             SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size, self.seq_length]							,							self.type_vocab_size		)

       SCREAMING_SNAKE_CASE_      		=     None
       if self.use_mc_token_ids:
             SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size, self.num_choices]							,							self.seq_length		)

       SCREAMING_SNAKE_CASE_      		=     None
       SCREAMING_SNAKE_CASE_      		=     None
       SCREAMING_SNAKE_CASE_      		=     None
       if self.use_labels:
             SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size]							,							self.type_sequence_label_size		)
             SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size, self.seq_length]							,							self.num_labels		)
             SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size]							,							self.num_choices		)

       SCREAMING_SNAKE_CASE_      		=     self.get_config()

       SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.num_hidden_layers, self.num_attention_heads]							,							2		)

       return (
           config,
           input_ids,
           input_mask,
           head_mask,
           token_type_ids,
           mc_token_ids,
           sequence_labels,
           token_labels,
           choice_labels,
       )



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       return CTRLConfig(
           vocab_size=self.vocab_size							,							n_embd=self.hidden_size							,							n_layer=self.num_hidden_layers							,							n_head=self.num_attention_heads							,							n_positions=self.max_position_embeddings							,							pad_token_id=self.pad_token_id							,							)



 def  lowerCAmelCase_    (				self    :						str							,							_lowerCAmelCase    :						str							,							_lowerCAmelCase    :						Optional[int]							,							_lowerCAmelCase    :						List[str]							,							_lowerCAmelCase    :						str							,							_lowerCAmelCase    :						Optional[int]							,							*_lowerCAmelCase    :						int		):
       SCREAMING_SNAKE_CASE_      		=     CTRLModel(config=_lowerCAmelCase		)
       model.to(_lowerCAmelCase		)
       model.eval()

       model(_lowerCAmelCase							,							token_type_ids=_lowerCAmelCase							,							head_mask=_lowerCAmelCase		)
       model(_lowerCAmelCase							,							token_type_ids=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     model(_lowerCAmelCase		)
       self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size)		)
       self.parent.assertEqual(len(result.past_key_values		)							,							config.n_layer		)



 def  lowerCAmelCase_    (				self    :						str							,							_lowerCAmelCase    :						Any							,							_lowerCAmelCase    :						Any							,							_lowerCAmelCase    :						int							,							_lowerCAmelCase    :						Union[str, Any]							,							_lowerCAmelCase    :						Dict							,							*_lowerCAmelCase    :						Union[str, Any]		):
       SCREAMING_SNAKE_CASE_      		=     CTRLLMHeadModel(_lowerCAmelCase		)
       model.to(_lowerCAmelCase		)
       model.eval()

       SCREAMING_SNAKE_CASE_      		=     model(_lowerCAmelCase							,							token_type_ids=_lowerCAmelCase							,							labels=_lowerCAmelCase		)
       self.parent.assertEqual(result.loss.shape							,							()		)
       self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)		)



 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     self.prepare_config_and_inputs()

       (
           (
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					(
           SCREAMING_SNAKE_CASE_
       )		,					
       )      		=     config_and_inputs

       SCREAMING_SNAKE_CASE_      		=     {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'head_mask': head_mask}

       return config, inputs_dict




 def  lowerCAmelCase_    (				self    :						Union[str, Any]							,							_lowerCAmelCase    :						Any							,							_lowerCAmelCase    :						Optional[Any]							,							_lowerCAmelCase    :						Tuple							,							_lowerCAmelCase    :						str							,							*_lowerCAmelCase    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     self.num_labels
       SCREAMING_SNAKE_CASE_      		=     CTRLForSequenceClassification(_lowerCAmelCase		)
       model.to(_lowerCAmelCase		)
       model.eval()
       SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size]							,							self.type_sequence_label_size		)
       SCREAMING_SNAKE_CASE_      		=     model(_lowerCAmelCase							,							token_type_ids=_lowerCAmelCase							,							labels=_lowerCAmelCase		)
       self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_labels)		)





@require_torch
class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE			,  _SCREAMING_SNAKE_CASE			,  _SCREAMING_SNAKE_CASE			,  unittest.TestCase   ):


 '''simple docstring'''







 lowercase_							       =       (CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
 lowercase_							       =       (CTRLLMHeadModel,) if is_torch_available() else ()
 lowercase_							       =       (
     {
         "feature-extraction": CTRLModel,
         "text-classification": CTRLForSequenceClassification,
         "text-generation": CTRLLMHeadModel,
         "zero-shot": CTRLForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 lowercase_							       =       True
 lowercase_							       =       False
 lowercase_							       =       False



 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						List[str]							,							_lowerCAmelCase    :						Union[str, Any]							,							_lowerCAmelCase    :						str							,							_lowerCAmelCase    :						Optional[Any]							,							_lowerCAmelCase    :						Optional[Any]		):
       if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
             # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
             # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
             # config could not be created.
             return True

       return False



 def  lowerCAmelCase_    (				self    :						Tuple		):
       SCREAMING_SNAKE_CASE_      		=     CTRLModelTester(self		)
       SCREAMING_SNAKE_CASE_      		=     ConfigTester(self							,							config_class=_lowerCAmelCase							,							n_embd=37		)



 def  lowerCAmelCase_    (				self    :						int		):
       super().tearDown()
       # clean-up as much as possible GPU memory occupied by PyTorch
       gc.collect()
       torch.cuda.empty_cache()



 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       self.config_tester.run_common_tests()



 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_ctrl_model(*_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_lm_head_model(*_lowerCAmelCase		)



 @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'		)
 def  lowerCAmelCase_    (				self    :						int		):
       pass



 @slow
 def  lowerCAmelCase_    (				self    :						str		):
       for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             SCREAMING_SNAKE_CASE_      		=     CTRLModel.from_pretrained(_lowerCAmelCase		)
             self.assertIsNotNone(_lowerCAmelCase		)




 @unittest.skip('The model doesn\'t support left padding'		)  # and it's not used enough to be worth fixing :)
 def  lowerCAmelCase_    (				self    :						int		):
       pass





@require_torch
class 			lowerCamelCase_				( unittest.TestCase   ):


 '''simple docstring'''







 def  lowerCAmelCase_    (				self    :						str		):
       super().tearDown()
       # clean-up as much as possible GPU memory occupied by PyTorch
       gc.collect()
       torch.cuda.empty_cache()




 @slow
 def  lowerCAmelCase_    (				self    :						int		):
       SCREAMING_SNAKE_CASE_      		=     CTRLLMHeadModel.from_pretrained('ctrl'		)
       model.to(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     torch.tensor(
           [[11_859, 0, 1_611, 8]]							,							dtype=torch.long							,							device=_lowerCAmelCase		)  # Legal the president is
       SCREAMING_SNAKE_CASE_      		=     [
           11_859,
           0,
           1_611,
           8,
           5,
           150,
           26_449,
           2,
           19,
           348,
           469,
           3,
           2_595,
           48,
           20_740,
           246_533,
           246_533,
           19,
           30,
           5,
       ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

       SCREAMING_SNAKE_CASE_      		=     model.generate(_lowerCAmelCase							,							do_sample=_lowerCAmelCase		)
       self.assertListEqual(output_ids[0].tolist()							,							_lowerCAmelCase		)



"""simple docstring"""




import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


__magic_name__						: Optional[Any]    =				'http://www.mocksite.com/file1.txt'
__magic_name__						: Tuple    =				'"text": ["foo", "foo"]'
__magic_name__						: str    =				'6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'




class 			__snake_case   :
							__a					 =       200
							__a					 =       {'''Content-Length''': '''100'''}
							__a					 =       {}




							def   __a   (     self:    List[str]     ,							**A_:    List[Any]   ):
										return [bytes(A_     ,							"""utf-8"""   )]





def       a_ (	*lowercase__ :List[Any],			**lowercase__ :Any     ):
			return MockResponse()





@pytest.mark.parametrize("""urls_type""",			[str, list, dict]     )
def       a_ (	lowercase__ :Optional[int],			lowercase__ :Any,			lowercase__ :Optional[int]     ):
			import requests

			monkeypatch.setattr(lowercase__,			"""request""",			lowercase__     )

			__lowerCamelCase          =      URL
			if issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      url
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      [url]
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      {"""train""": url}
			__lowerCamelCase          =      """dummy"""
			__lowerCamelCase          =      """downloads"""
			__lowerCamelCase          =      tmp_path
			__lowerCamelCase          =      DownloadConfig(
			    cache_dir=os.path.join(lowercase__,			lowercase__     ),			use_etag=lowercase__,			)
			__lowerCamelCase          =      DownloadManager(dataset_name=lowercase__,			download_config=lowercase__     )
			__lowerCamelCase          =      dl_manager.download(lowercase__     )
			__lowerCamelCase          =      urls
			for downloaded_paths in [downloaded_paths]:
						if isinstance(lowercase__,			lowercase__     ):
									__lowerCamelCase          =      [downloaded_paths]
									__lowerCamelCase          =      [urls]
						elif isinstance(lowercase__,			lowercase__     ):
									assert "train" in downloaded_paths.keys()
									__lowerCamelCase          =      downloaded_paths.values()
									__lowerCamelCase          =      urls.values()
						assert downloaded_paths
						for downloaded_path, input_url in zip(lowercase__,			lowercase__     ):
									assert downloaded_path == dl_manager.downloaded_paths[input_url]
									__lowerCamelCase          =      Path(lowercase__     )
									__lowerCamelCase          =      downloaded_path.parts
									assert parts[-1] == HASH
									assert parts[-2] == cache_subdir
									assert downloaded_path.exists()
									__lowerCamelCase          =      downloaded_path.read_text()
									assert content == CONTENT
									__lowerCamelCase          =      downloaded_path.with_suffix(""".json"""     )
									assert metadata_downloaded_path.exists()
									__lowerCamelCase          =      json.loads(metadata_downloaded_path.read_text()     )
									assert metadata_content == {"url": URL, "etag": None}





@pytest.mark.parametrize("""paths_type""",			[str, list, dict]     )
def       a_ (	lowercase__ :Dict,			lowercase__ :Optional[Any],			lowercase__ :Dict     ):
			__lowerCamelCase          =      str(lowercase__     )
			if issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      filename
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      [filename]
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      {"""train""": filename}
			__lowerCamelCase          =      """dummy"""
			__lowerCamelCase          =      xz_file.parent
			__lowerCamelCase          =      """extracted"""
			__lowerCamelCase          =      DownloadConfig(
			    cache_dir=lowercase__,			use_etag=lowercase__,			)
			__lowerCamelCase          =      DownloadManager(dataset_name=lowercase__,			download_config=lowercase__     )
			__lowerCamelCase          =      dl_manager.extract(lowercase__     )
			__lowerCamelCase          =      paths
			for extracted_paths in [extracted_paths]:
						if isinstance(lowercase__,			lowercase__     ):
									__lowerCamelCase          =      [extracted_paths]
									__lowerCamelCase          =      [paths]
						elif isinstance(lowercase__,			lowercase__     ):
									assert "train" in extracted_paths.keys()
									__lowerCamelCase          =      extracted_paths.values()
									__lowerCamelCase          =      paths.values()
						assert extracted_paths
						for extracted_path, input_path in zip(lowercase__,			lowercase__     ):
									assert extracted_path == dl_manager.extracted_paths[input_path]
									__lowerCamelCase          =      Path(lowercase__     )
									__lowerCamelCase          =      extracted_path.parts
									assert parts[-1] == hash_url_to_filename(lowercase__,			etag=lowercase__     )
									assert parts[-2] == extracted_subdir
									assert extracted_path.exists()
									__lowerCamelCase          =      extracted_path.read_text()
									__lowerCamelCase          =      text_file.read_text()
									assert extracted_file_content == expected_file_content





def       a_ (	lowercase__ :List[str],			lowercase__ :int     ):
			assert path.endswith(""".jsonl"""     )
			for num_items, line in enumerate(lowercase__,			start=1     ):
						__lowerCamelCase          =      json.loads(line.decode("""utf-8"""     )     )
						assert item.keys() == {"col_1", "col_2", "col_3"}
			assert num_items == 4





@pytest.mark.parametrize("""archive_jsonl""",			["""tar_jsonl_path""", """zip_jsonl_path"""]     )
def       a_ (	lowercase__ :Optional[int],			lowercase__ :Union[str, Any]     ):
			__lowerCamelCase          =      request.getfixturevalue(lowercase__     )
			__lowerCamelCase          =      DownloadManager()
			for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(lowercase__     ),			start=1     ):
						_test_jsonl(lowercase__,			lowercase__     )
			assert num_jsonl == 2





@pytest.mark.parametrize("""archive_nested_jsonl""",			["""tar_nested_jsonl_path""", """zip_nested_jsonl_path"""]     )
def       a_ (	lowercase__ :Optional[int],			lowercase__ :List[Any]     ):
			__lowerCamelCase          =      request.getfixturevalue(lowercase__     )
			__lowerCamelCase          =      DownloadManager()
			for num_tar, (path, file) in enumerate(dl_manager.iter_archive(lowercase__     ),			start=1     ):
						for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(lowercase__     ),			start=1     ):
									_test_jsonl(lowercase__,			lowercase__     )
			assert num_tar == 1
			assert num_jsonl == 2





def       a_ (	lowercase__ :Tuple     ):
			__lowerCamelCase          =      DownloadManager()
			for num_file, file in enumerate(dl_manager.iter_files(lowercase__     ),			start=1     ):
						assert os.path.basename(lowercase__     ) == ("test.txt" if num_file == 1 else "train.txt")
			assert num_file == 2




