





import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
							import jax.numpy as jnp

if is_tf_available():
							import tensorflow as tf

if is_torch_available():
							import torch





class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   {
										    '''task_specific_params''': {
										        '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 128, '''min_length''': 12, '''num_beams''': 4},
										        '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 142, '''min_length''': 56, '''num_beams''': 4},
										        '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 62, '''min_length''': 11, '''num_beams''': 6},
										    }
										}
										__SCREAMING_SNAKE_CASE:				Dict           =   {
										    '''task_specific_params.summarization.length_penalty''': 1.0,
										    '''task_specific_params.summarization.max_length''': 128,
										    '''task_specific_params.summarization.min_length''': 12,
										    '''task_specific_params.summarization.num_beams''': 4,
										    '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
										    '''task_specific_params.summarization_cnn.max_length''': 142,
										    '''task_specific_params.summarization_cnn.min_length''': 56,
										    '''task_specific_params.summarization_cnn.num_beams''': 4,
										    '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
										    '''task_specific_params.summarization_xsum.max_length''': 62,
										    '''task_specific_params.summarization_xsum.min_length''': 11,
										    '''task_specific_params.summarization_xsum.num_beams''': 6,
										}

										self.assertEqual(flatten_dict(_lowerCAmelCase					)			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   np.random.randn(3			, 4					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase					)			, x.transpose()					)					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   np.random.randn(3			, 4			, 5					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase			, axes=(1, 2, 0)					)			, x.transpose((1, 2, 0)					)					)					)






				@require_torch
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase					)			, transpose(_lowerCAmelCase					).numpy()					)					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   np.random.randn(3			, 4			, 5					)
										__SCREAMING_SNAKE_CASE:				int           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase			, axes=(1, 2, 0)					)			, transpose(_lowerCAmelCase			, axes=(1, 2, 0)					).numpy()					)					)






				@require_tf
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Dict           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase					)			, transpose(_lowerCAmelCase					).numpy()					)					)

										__SCREAMING_SNAKE_CASE:				Dict           =   np.random.randn(3			, 4			, 5					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase			, axes=(1, 2, 0)					)			, transpose(_lowerCAmelCase			, axes=(1, 2, 0)					).numpy()					)					)






				@require_flax
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase					)			, np.asarray(transpose(_lowerCAmelCase					)					)					)					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   np.random.randn(3			, 4			, 5					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(transpose(_lowerCAmelCase			, axes=(1, 2, 0)					)			, np.asarray(transpose(_lowerCAmelCase			, axes=(1, 2, 0)					)					)					)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   np.random.randn(3			, 4					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (4, 3)					)			, np.reshape(_lowerCAmelCase			, (4, 3)					)					)					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   np.random.randn(3			, 4			, 5					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (12, 5)					)			, np.reshape(_lowerCAmelCase			, (12, 5)					)					)					)






				@require_torch
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (4, 3)					)			, reshape(_lowerCAmelCase			, (4, 3)					).numpy()					)					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   np.random.randn(3			, 4			, 5					)
										__SCREAMING_SNAKE_CASE:				Any           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (12, 5)					)			, reshape(_lowerCAmelCase			, (12, 5)					).numpy()					)					)






				@require_tf
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (4, 3)					)			, reshape(_lowerCAmelCase			, (4, 3)					).numpy()					)					)

										__SCREAMING_SNAKE_CASE:				Tuple           =   np.random.randn(3			, 4			, 5					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (12, 5)					)			, reshape(_lowerCAmelCase			, (12, 5)					).numpy()					)					)






				@require_flax
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Any           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (4, 3)					)			, np.asarray(reshape(_lowerCAmelCase			, (4, 3)					)					)					)					)

										__SCREAMING_SNAKE_CASE:				str           =   np.random.randn(3			, 4			, 5					)
										__SCREAMING_SNAKE_CASE:				Any           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(reshape(_lowerCAmelCase			, (12, 5)					)			, np.asarray(reshape(_lowerCAmelCase			, (12, 5)					)					)					)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   np.random.randn(1			, 3			, 4					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase					)			, np.squeeze(_lowerCAmelCase					)					)					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   np.random.randn(1			, 4			, 1			, 5					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase			, axis=2					)			, np.squeeze(_lowerCAmelCase			, axis=2					)					)					)






				@require_torch
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   np.random.randn(1			, 3			, 4					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase					)			, squeeze(_lowerCAmelCase					).numpy()					)					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.random.randn(1			, 4			, 1			, 5					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase			, axis=2					)			, squeeze(_lowerCAmelCase			, axis=2					).numpy()					)					)






				@require_tf
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   np.random.randn(1			, 3			, 4					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase					)			, squeeze(_lowerCAmelCase					).numpy()					)					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   np.random.randn(1			, 4			, 1			, 5					)
										__SCREAMING_SNAKE_CASE:				Any           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase			, axis=2					)			, squeeze(_lowerCAmelCase			, axis=2					).numpy()					)					)






				@require_flax
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.random.randn(1			, 3			, 4					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase					)			, np.asarray(squeeze(_lowerCAmelCase					)					)					)					)

										__SCREAMING_SNAKE_CASE:				Any           =   np.random.randn(1			, 4			, 1			, 5					)
										__SCREAMING_SNAKE_CASE:				Any           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(squeeze(_lowerCAmelCase			, axis=2					)			, np.asarray(squeeze(_lowerCAmelCase			, axis=2					)					)					)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   np.random.randn(3			, 4					)
										self.assertTrue(np.allclose(expand_dims(_lowerCAmelCase			, axis=1					)			, np.expand_dims(_lowerCAmelCase			, axis=1					)					)					)






				@require_torch
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   torch.tensor(_lowerCAmelCase					)
										self.assertTrue(np.allclose(expand_dims(_lowerCAmelCase			, axis=1					)			, expand_dims(_lowerCAmelCase			, axis=1					).numpy()					)					)






				@require_tf
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   tf.constant(_lowerCAmelCase					)
										self.assertTrue(np.allclose(expand_dims(_lowerCAmelCase			, axis=1					)			, expand_dims(_lowerCAmelCase			, axis=1					).numpy()					)					)






				@require_flax
				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   np.random.randn(3			, 4					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   jnp.array(_lowerCAmelCase					)
										self.assertTrue(np.allclose(expand_dims(_lowerCAmelCase			, axis=1					)			, np.asarray(expand_dims(_lowerCAmelCase			, axis=1					)					)					)					)










import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCAmelCase							:       Dict					 =	"""\
    Text data.
    Second line of data."""

lowerCAmelCase							:       str					 =	"""file"""






@pytest.fixture(scope='''session'''					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Union[str, Any]					)      ->    Any:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Any           =   tmp_path_factory.mktemp('''data'''					) / (FILE_PATH + '''.zstd''')
						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   bytes(UpperCamelCase__       ,			'''utf-8'''					)
						with zstd.open(UpperCamelCase__       ,			'''wb'''					) as f:
												f.write(UpperCamelCase__					)
						return path






@pytest.fixture
def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[Any]					)      ->    int:

						"""simple docstring"""





						with open(os.path.join(tmpfs.local_root_dir       ,			UpperCamelCase__					)       ,			'''w'''					) as f:
												f.write(UpperCamelCase__					)
						return FILE_PATH






@pytest.mark.parametrize('''compression_format'''       ,			['''gzip''', '''xz''', '''zstd''']					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[int]       ,			UpperCamelCase__   :   List[str]       ,			UpperCamelCase__   :   Dict       ,			UpperCamelCase__   :   Optional[Any]       ,			UpperCamelCase__   :   Dict       ,			UpperCamelCase__   :   List[Any]					)      ->    Any:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Dict           =   {'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
						__SCREAMING_SNAKE_CASE:				List[str]           =   input_paths[compression_format]
						__SCREAMING_SNAKE_CASE:				List[Any]           =   tmp_path / '''cache'''
						__SCREAMING_SNAKE_CASE:				List[str]           =   DownloadConfig(cache_dir=UpperCamelCase__       ,			extract_compressed_file=UpperCamelCase__					)
						__SCREAMING_SNAKE_CASE:				Tuple           =   cached_path(UpperCamelCase__       ,			download_config=UpperCamelCase__					)
						with open(UpperCamelCase__					) as f:
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   f.read()
						with open(UpperCamelCase__					) as f:
												__SCREAMING_SNAKE_CASE:				Dict           =   f.read()
						assert extracted_file_content == expected_file_content






@pytest.mark.parametrize('''default_extracted'''       ,			[True, False]					)
@pytest.mark.parametrize('''default_cache_dir'''       ,			[True, False]					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Tuple       ,			UpperCamelCase__   :   List[str]       ,			UpperCamelCase__   :   str       ,			UpperCamelCase__   :   Optional[int]       ,			UpperCamelCase__   :   Optional[Any]					)      ->    Optional[Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Tuple           =   '''custom_cache'''
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   '''custom_extracted_dir'''
						__SCREAMING_SNAKE_CASE:				List[Any]           =   tmp_path / '''custom_extracted_path'''
						if default_extracted:
												__SCREAMING_SNAKE_CASE:				int           =   ('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
						else:
												monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''       ,			UpperCamelCase__					)
												monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''       ,			str(UpperCamelCase__					)					)
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

						__SCREAMING_SNAKE_CASE:				Dict           =   xz_file
						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   (
						    DownloadConfig(extract_compressed_file=UpperCamelCase__					)
						    if default_cache_dir
						    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir       ,			extract_compressed_file=UpperCamelCase__					)
						)
						__SCREAMING_SNAKE_CASE:				int           =   cached_path(UpperCamelCase__       ,			download_config=UpperCamelCase__					)
						assert Path(UpperCamelCase__					).parent.parts[-2:] == expected






def 		lowerCAmelCase  (	UpperCamelCase__   :   Dict					)      ->    Union[str, Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   str(Path(UpperCamelCase__					).resolve()					)
						assert cached_path(UpperCamelCase__					) == text_file
						# relative path
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   str(Path(UpperCamelCase__					).resolve().relative_to(Path(os.getcwd()					)					)					)
						assert cached_path(UpperCamelCase__					) == text_file






def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[Any]					)      ->    Tuple:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				List[Any]           =   str(tmp_path.resolve() / '''__missing_file__.txt'''					)
						with pytest.raises(UpperCamelCase__					):
												cached_path(UpperCamelCase__					)
						# relative path
						__SCREAMING_SNAKE_CASE:				int           =   '''./__missing_file__.txt'''
						with pytest.raises(UpperCamelCase__					):
												cached_path(UpperCamelCase__					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   Tuple					)      ->    Union[str, Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Tuple           =   get_from_cache(F"""tmp://{tmpfs_file}"""					)
						with open(UpperCamelCase__					) as f:
												__SCREAMING_SNAKE_CASE:				Tuple           =   f.read()
						assert output_file_content == FILE_CONTENT






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	)      ->    Any:

						"""simple docstring"""





						with pytest.raises(UpperCamelCase__					):
												cached_path('''https://huggingface.co'''					)






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   int					)      ->    Union[str, Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Optional[int]           =   tmp_path_factory.mktemp('''data'''					) / '''file.html'''
						with pytest.raises(UpperCamelCase__					):
												http_get('''https://huggingface.co'''       ,			temp_file=UpperCamelCase__					)
						with pytest.raises(UpperCamelCase__					):
												http_head('''https://huggingface.co'''					)






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Dict					)      ->    List[str]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				List[Any]           =   tmp_path_factory.mktemp('''data'''					) / '''file.html'''
						with pytest.raises(UpperCamelCase__					):
												ftp_get('''ftp://huggingface.co'''       ,			temp_file=UpperCamelCase__					)
						with pytest.raises(UpperCamelCase__					):
												ftp_head('''ftp://huggingface.co'''					)






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[int]					)      ->    Tuple:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Tuple           =   tmp_path_factory.mktemp('''data'''					) / '''file.html'''
						with pytest.raises(UpperCamelCase__					):
												fsspec_get('''s3://huggingface.co'''       ,			temp_file=UpperCamelCase__					)
						with pytest.raises(UpperCamelCase__					):
												fsspec_head('''s3://huggingface.co'''					)



