


"""simple docstring"""




import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


__magic_name__						: Optional[Any]    =				{
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 1_2_8,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 5_0,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 1_0,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 1_0,
    'exponential_decay_length_penalty': (5, 1.0_1),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}




@is_staging_test
class 			__snake_case   (unittest.TestCase  ):




							@classmethod
							def   __a   (     cls:    List[Any]   ):
										__lowerCamelCase          =      TOKEN
										HfFolder.save_token(A_   )




							@classmethod
							def   __a   (     cls:    Optional[int]   ):
										try:
													delete_repo(token=cls._token     ,							repo_id="""test-config"""   )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token     ,							repo_id="""valid_org/test-config-org"""   )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token     ,							repo_id="""test-dynamic-config"""   )
										except HTTPError:
													pass




							def   __a   (     self:    Dict   ):
										__lowerCamelCase          =      BertConfig(
										    vocab_size=99     ,							hidden_size=32     ,							num_hidden_layers=5     ,							num_attention_heads=4     ,							intermediate_size=37   )
										config.push_to_hub("""test-config"""     ,							use_auth_token=self._token   )

										__lowerCamelCase          =      BertConfig.from_pretrained(f'{USER}/test-config'   )
										for k, v in config.to_dict().items():
													if k != "transformers_version":
																self.assertEqual(A_     ,							getattr(A_     ,							A_   )   )

        # Reset repo
										delete_repo(token=self._token     ,							repo_id="""test-config"""   )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													config.save_pretrained(A_     ,							repo_id="""test-config"""     ,							push_to_hub=A_     ,							use_auth_token=self._token   )

										__lowerCamelCase          =      BertConfig.from_pretrained(f'{USER}/test-config'   )
										for k, v in config.to_dict().items():
													if k != "transformers_version":
																self.assertEqual(A_     ,							getattr(A_     ,							A_   )   )




							def   __a   (     self:    List[Any]   ):
										__lowerCamelCase          =      BertConfig(
										    vocab_size=99     ,							hidden_size=32     ,							num_hidden_layers=5     ,							num_attention_heads=4     ,							intermediate_size=37   )
										config.push_to_hub("""valid_org/test-config-org"""     ,							use_auth_token=self._token   )

										__lowerCamelCase          =      BertConfig.from_pretrained("""valid_org/test-config-org"""   )
										for k, v in config.to_dict().items():
													if k != "transformers_version":
																self.assertEqual(A_     ,							getattr(A_     ,							A_   )   )

        # Reset repo
										delete_repo(token=self._token     ,							repo_id="""valid_org/test-config-org"""   )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													config.save_pretrained(
													    A_     ,							repo_id="""valid_org/test-config-org"""     ,							push_to_hub=A_     ,							use_auth_token=self._token   )

										__lowerCamelCase          =      BertConfig.from_pretrained("""valid_org/test-config-org"""   )
										for k, v in config.to_dict().items():
													if k != "transformers_version":
																self.assertEqual(A_     ,							getattr(A_     ,							A_   )   )




							def   __a   (     self:    Optional[int]   ):
										CustomConfig.register_for_auto_class()
										__lowerCamelCase          =      CustomConfig(attribute=42   )

										config.push_to_hub("""test-dynamic-config"""     ,							use_auth_token=self._token   )

										# This has added the proper auto_map field to the config
										self.assertDictEqual(config.auto_map     ,							{"""AutoConfig""": """custom_configuration.CustomConfig"""}   )

										__lowerCamelCase          =      AutoConfig.from_pretrained(f'{USER}/test-dynamic-config'     ,							trust_remote_code=A_   )
										# Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
										self.assertEqual(new_config.__class__.__name__     ,							"""CustomConfig"""   )
										self.assertEqual(new_config.attribute     ,							42   )




class 			__snake_case   (unittest.TestCase  ):




							def   __a   (     self:    List[str]   ):
										__lowerCamelCase          =      GPTaConfig()

										# attempt to modify each of int/float/bool/str config records and verify they were updated
										__lowerCamelCase          =      c.n_embd + 1  # int
										__lowerCamelCase          =      c.resid_pdrop + 1.0  # float
										__lowerCamelCase          =      not c.scale_attn_weights  # bool
										__lowerCamelCase          =      c.summary_type + """foo"""  # str
										c.update_from_string(
										    f'n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}'   )
										self.assertEqual(A_     ,							c.n_embd     ,							"""mismatch for key: n_embd"""   )
										self.assertEqual(A_     ,							c.resid_pdrop     ,							"""mismatch for key: resid_pdrop"""   )
										self.assertEqual(A_     ,							c.scale_attn_weights     ,							"""mismatch for key: scale_attn_weights"""   )
										self.assertEqual(A_     ,							c.summary_type     ,							"""mismatch for key: summary_type"""   )




							def   __a   (     self:    Optional[int]   ):
										__lowerCamelCase          =      PretrainedConfig()
										__lowerCamelCase          =      [key for key in base_config.__dict__ if key not in config_common_kwargs]
										# If this part of the test fails, you have arguments to addin config_common_kwargs above.
										self.assertListEqual(
										    A_     ,							["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]   )
										__lowerCamelCase          =      [key for key, value in config_common_kwargs.items() if value == getattr(A_     ,							A_   )]
										if len(A_   ) > 0:
													raise ValueError(
													    """The following keys are set with the default values in"""
													    """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
													    f' {", ".join(A_   )}.'   )




							def   __a   (     self:    Dict   ):
										with self.assertRaises(A_   ):
													# config is in subfolder, the following should not work without specifying the subfolder
													__lowerCamelCase          =      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""   )

										__lowerCamelCase          =      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""     ,							subfolder="""bert"""   )

										self.assertIsNotNone(A_   )




							def   __a   (     self:    Optional[Any]   ):
										# A mock response for an HTTP head request to emulate server down
										__lowerCamelCase          =      mock.Mock()
										__lowerCamelCase          =      5_00
										__lowerCamelCase          =      {}
										__lowerCamelCase          =      HTTPError
										__lowerCamelCase          =      {}

										# Download this model to make sure it's in the cache.
										__lowerCamelCase          =      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )

										# Under the mock environment we get a 500 error when trying to reach the model.
										with mock.patch("""requests.Session.request"""     ,							return_value=A_   ) as mock_head:
													__lowerCamelCase          =      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )
													# This check we did call the fake head request
													mock_head.assert_called()




							def   __a   (     self:    Union[str, Any]   ):
										# This test is for deprecated behavior and can be removed in v5
										__lowerCamelCase          =      BertConfig.from_pretrained(
										    """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""   )




							def   __a   (     self:    Optional[int]   ):
										__lowerCamelCase          =      AutoConfig.from_pretrained("""bert-base-cased"""   )
										__lowerCamelCase          =      ["""config.4.0.0.json"""]

										with tempfile.TemporaryDirectory() as tmp_dir:
													configuration.save_pretrained(A_   )
													__lowerCamelCase          =      2
													json.dump(configuration.to_dict()     ,							open(os.path.join(A_     ,							"""config.4.0.0.json"""   )     ,							"""w"""   )   )

													# This should pick the new configuration file as the version of Transformers is > 4.0.0
													__lowerCamelCase          =      AutoConfig.from_pretrained(A_   )
													self.assertEqual(new_configuration.hidden_size     ,							2   )

													# Will need to be adjusted if we reach v42 and this test is still here.
													# Should pick the old configuration file as the version of Transformers is < 4.42.0
													__lowerCamelCase          =      ["""config.42.0.0.json"""]
													__lowerCamelCase          =      7_68
													configuration.save_pretrained(A_   )
													shutil.move(os.path.join(A_     ,							"""config.4.0.0.json"""   )     ,							os.path.join(A_     ,							"""config.42.0.0.json"""   )   )
													__lowerCamelCase          =      AutoConfig.from_pretrained(A_   )
													self.assertEqual(new_configuration.hidden_size     ,							7_68   )




							def   __a   (     self:    Union[str, Any]   ):
										# This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
										__lowerCamelCase          =      """hf-internal-testing/test-two-configs"""

										import transformers as new_transformers

										__lowerCamelCase          =      """v4.0.0"""
										__lowerCamelCase     ,__lowerCamelCase          =      new_transformers.models.auto.AutoConfig.from_pretrained(
										    A_     ,							return_unused_kwargs=A_   )
										self.assertEqual(new_configuration.hidden_size     ,							2   )
										# This checks `_configuration_file` ia not kept in the kwargs by mistake.
										self.assertDictEqual(A_     ,							{}   )

										# Testing an older version by monkey-patching the version in the module it's used.
										import transformers as old_transformers

										__lowerCamelCase          =      """v3.0.0"""
										__lowerCamelCase          =      old_transformers.models.auto.AutoConfig.from_pretrained(A_   )
										self.assertEqual(old_configuration.hidden_size     ,							7_68   )








"""simple docstring"""




import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


__magic_name__						: Optional[Any]    =				'http://www.mocksite.com/file1.txt'
__magic_name__						: Tuple    =				'"text": ["foo", "foo"]'
__magic_name__						: str    =				'6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'




class 			__snake_case   :
							__a					 =       200
							__a					 =       {'''Content-Length''': '''100'''}
							__a					 =       {}




							def   __a   (     self:    List[str]     ,							**A_:    List[Any]   ):
										return [bytes(A_     ,							"""utf-8"""   )]





def       a_ (	*lowercase__ :List[Any],			**lowercase__ :Any     ):
			return MockResponse()





@pytest.mark.parametrize("""urls_type""",			[str, list, dict]     )
def       a_ (	lowercase__ :Optional[int],			lowercase__ :Any,			lowercase__ :Optional[int]     ):
			import requests

			monkeypatch.setattr(lowercase__,			"""request""",			lowercase__     )

			__lowerCamelCase          =      URL
			if issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      url
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      [url]
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      {"""train""": url}
			__lowerCamelCase          =      """dummy"""
			__lowerCamelCase          =      """downloads"""
			__lowerCamelCase          =      tmp_path
			__lowerCamelCase          =      DownloadConfig(
			    cache_dir=os.path.join(lowercase__,			lowercase__     ),			use_etag=lowercase__,			)
			__lowerCamelCase          =      DownloadManager(dataset_name=lowercase__,			download_config=lowercase__     )
			__lowerCamelCase          =      dl_manager.download(lowercase__     )
			__lowerCamelCase          =      urls
			for downloaded_paths in [downloaded_paths]:
						if isinstance(lowercase__,			lowercase__     ):
									__lowerCamelCase          =      [downloaded_paths]
									__lowerCamelCase          =      [urls]
						elif isinstance(lowercase__,			lowercase__     ):
									assert "train" in downloaded_paths.keys()
									__lowerCamelCase          =      downloaded_paths.values()
									__lowerCamelCase          =      urls.values()
						assert downloaded_paths
						for downloaded_path, input_url in zip(lowercase__,			lowercase__     ):
									assert downloaded_path == dl_manager.downloaded_paths[input_url]
									__lowerCamelCase          =      Path(lowercase__     )
									__lowerCamelCase          =      downloaded_path.parts
									assert parts[-1] == HASH
									assert parts[-2] == cache_subdir
									assert downloaded_path.exists()
									__lowerCamelCase          =      downloaded_path.read_text()
									assert content == CONTENT
									__lowerCamelCase          =      downloaded_path.with_suffix(""".json"""     )
									assert metadata_downloaded_path.exists()
									__lowerCamelCase          =      json.loads(metadata_downloaded_path.read_text()     )
									assert metadata_content == {"url": URL, "etag": None}





@pytest.mark.parametrize("""paths_type""",			[str, list, dict]     )
def       a_ (	lowercase__ :Dict,			lowercase__ :Optional[Any],			lowercase__ :Dict     ):
			__lowerCamelCase          =      str(lowercase__     )
			if issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      filename
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      [filename]
			elif issubclass(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      {"""train""": filename}
			__lowerCamelCase          =      """dummy"""
			__lowerCamelCase          =      xz_file.parent
			__lowerCamelCase          =      """extracted"""
			__lowerCamelCase          =      DownloadConfig(
			    cache_dir=lowercase__,			use_etag=lowercase__,			)
			__lowerCamelCase          =      DownloadManager(dataset_name=lowercase__,			download_config=lowercase__     )
			__lowerCamelCase          =      dl_manager.extract(lowercase__     )
			__lowerCamelCase          =      paths
			for extracted_paths in [extracted_paths]:
						if isinstance(lowercase__,			lowercase__     ):
									__lowerCamelCase          =      [extracted_paths]
									__lowerCamelCase          =      [paths]
						elif isinstance(lowercase__,			lowercase__     ):
									assert "train" in extracted_paths.keys()
									__lowerCamelCase          =      extracted_paths.values()
									__lowerCamelCase          =      paths.values()
						assert extracted_paths
						for extracted_path, input_path in zip(lowercase__,			lowercase__     ):
									assert extracted_path == dl_manager.extracted_paths[input_path]
									__lowerCamelCase          =      Path(lowercase__     )
									__lowerCamelCase          =      extracted_path.parts
									assert parts[-1] == hash_url_to_filename(lowercase__,			etag=lowercase__     )
									assert parts[-2] == extracted_subdir
									assert extracted_path.exists()
									__lowerCamelCase          =      extracted_path.read_text()
									__lowerCamelCase          =      text_file.read_text()
									assert extracted_file_content == expected_file_content





def       a_ (	lowercase__ :List[str],			lowercase__ :int     ):
			assert path.endswith(""".jsonl"""     )
			for num_items, line in enumerate(lowercase__,			start=1     ):
						__lowerCamelCase          =      json.loads(line.decode("""utf-8"""     )     )
						assert item.keys() == {"col_1", "col_2", "col_3"}
			assert num_items == 4





@pytest.mark.parametrize("""archive_jsonl""",			["""tar_jsonl_path""", """zip_jsonl_path"""]     )
def       a_ (	lowercase__ :Optional[int],			lowercase__ :Union[str, Any]     ):
			__lowerCamelCase          =      request.getfixturevalue(lowercase__     )
			__lowerCamelCase          =      DownloadManager()
			for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(lowercase__     ),			start=1     ):
						_test_jsonl(lowercase__,			lowercase__     )
			assert num_jsonl == 2





@pytest.mark.parametrize("""archive_nested_jsonl""",			["""tar_nested_jsonl_path""", """zip_nested_jsonl_path"""]     )
def       a_ (	lowercase__ :Optional[int],			lowercase__ :List[Any]     ):
			__lowerCamelCase          =      request.getfixturevalue(lowercase__     )
			__lowerCamelCase          =      DownloadManager()
			for num_tar, (path, file) in enumerate(dl_manager.iter_archive(lowercase__     ),			start=1     ):
						for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(lowercase__     ),			start=1     ):
									_test_jsonl(lowercase__,			lowercase__     )
			assert num_tar == 1
			assert num_jsonl == 2





def       a_ (	lowercase__ :Tuple     ):
			__lowerCamelCase          =      DownloadManager()
			for num_file, file in enumerate(dl_manager.iter_files(lowercase__     ),			start=1     ):
						assert os.path.basename(lowercase__     ) == ("test.txt" if num_file == 1 else "train.txt")
			assert num_file == 2




