


'''simple docstring'''

import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCAmelCase :int             =     '\\n    Text data.\n    Second line of data.'

lowerCAmelCase :Optional[Any]             =     'file'





@pytest.fixture(scope='session'						)
def lowerCamelCase     (     lowerCAmelCase							:      Tuple						):





	"""simple docstring"""







	__magic_name__			:       Optional[int]	      =	tmp_path_factory.mktemp('data'						) / (FILE_PATH + '.zstd')
	__magic_name__			:       Tuple	      =	bytes(__UpperCAmelCase						,						'utf-8'						)
	with zstd.open(__UpperCAmelCase						,						'wb'						) as f:
		f.write(__UpperCAmelCase						)
	return path





@pytest.fixture
def lowerCamelCase     (     lowerCAmelCase							:      Optional[int]						):





	"""simple docstring"""







	with open(os.path.join(tmpfs.local_root_dir						,						__UpperCAmelCase						)						,						'w'						) as f:
		f.write(__UpperCAmelCase						)
	return FILE_PATH





@pytest.mark.parametrize('compression_format'						,						['gzip', 'xz', 'zstd']						)
def lowerCamelCase     (     lowerCAmelCase							:      Any						,						lowerCAmelCase							:      Optional[Any]						,						lowerCAmelCase							:      Tuple						,						lowerCAmelCase							:      Tuple						,						lowerCAmelCase							:      Tuple						,						lowerCAmelCase							:      Tuple						):





	"""simple docstring"""







	__magic_name__			:       Dict	      =	{'gzip': gz_file, 'xz': xz_file, 'zstd': zstd_path}
	__magic_name__			:       List[Any]	      =	input_paths[compression_format]
	__magic_name__			:       Any	      =	tmp_path / 'cache'
	__magic_name__			:       Optional[Any]	      =	DownloadConfig(cache_dir=__UpperCAmelCase						,						extract_compressed_file=__UpperCAmelCase						)
	__magic_name__			:       List[Any]	      =	cached_path(__UpperCAmelCase						,						download_config=__UpperCAmelCase						)
	with open(__UpperCAmelCase						) as f:
		__magic_name__			:       str	      =	f.read()
	with open(__UpperCAmelCase						) as f:
		__magic_name__			:       Tuple	      =	f.read()
	assert extracted_file_content == expected_file_content





@pytest.mark.parametrize('default_extracted'						,						[True, False]						)
@pytest.mark.parametrize('default_cache_dir'						,						[True, False]						)
def lowerCamelCase     (     lowerCAmelCase							:      str						,						lowerCAmelCase							:      Optional[Any]						,						lowerCAmelCase							:      Union[str, Any]						,						lowerCAmelCase							:      Union[str, Any]						,						lowerCAmelCase							:      List[str]						):





	"""simple docstring"""







	__magic_name__			:       Optional[int]	      =	'custom_cache'
	__magic_name__			:       Dict	      =	'custom_extracted_dir'
	__magic_name__			:       Tuple	      =	tmp_path / 'custom_extracted_path'
	if default_extracted:
		__magic_name__			:       Optional[int]	      =	('downloads' if default_cache_dir else custom_cache_dir, 'extracted')
	else:
		monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_DIR'						,						__UpperCAmelCase						)
		monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_PATH'						,						str(__UpperCAmelCase						)						)
		__magic_name__			:       Optional[Any]	      =	custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

	__magic_name__			:       Union[str, Any]	      =	xz_file
	__magic_name__			:       Dict	      =	(
	    DownloadConfig(extract_compressed_file=__UpperCAmelCase						)
	    if default_cache_dir
	    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir						,						extract_compressed_file=__UpperCAmelCase						)
	)
	__magic_name__			:       List[Any]	      =	cached_path(__UpperCAmelCase						,						download_config=__UpperCAmelCase						)
	assert Path(__UpperCAmelCase						).parent.parts[-2:] == expected





def lowerCamelCase     (     lowerCAmelCase							:      Any						):





	"""simple docstring"""







	__magic_name__			:       Tuple	      =	str(Path(__UpperCAmelCase						).resolve()						)
	assert cached_path(__UpperCAmelCase						) == text_file
	# relative path
	__magic_name__			:       Tuple	      =	str(Path(__UpperCAmelCase						).resolve().relative_to(Path(os.getcwd()						)						)						)
	assert cached_path(__UpperCAmelCase						) == text_file





def lowerCamelCase     (     lowerCAmelCase							:      Dict						):





	"""simple docstring"""







	__magic_name__			:       int	      =	str(tmp_path.resolve() / '__missing_file__.txt'						)
	with pytest.raises(__UpperCAmelCase						):
		cached_path(__UpperCAmelCase						)
	# relative path
	__magic_name__			:       List[str]	      =	'./__missing_file__.txt'
	with pytest.raises(__UpperCAmelCase						):
		cached_path(__UpperCAmelCase						)





def lowerCamelCase     (     lowerCAmelCase							:      List[str]						):





	"""simple docstring"""







	__magic_name__			:       Optional[int]	      =	get_from_cache(f'tmp://{tmpfs_file}'						)
	with open(__UpperCAmelCase						) as f:
		__magic_name__			:       Optional[int]	      =	f.read()
	assert output_file_content == FILE_CONTENT





@patch('datasets.config.HF_DATASETS_OFFLINE'						,						__UpperCAmelCase						)
def lowerCamelCase     (     ):





	"""simple docstring"""







	with pytest.raises(__UpperCAmelCase						):
		cached_path('https://huggingface.co'						)





@patch('datasets.config.HF_DATASETS_OFFLINE'						,						__UpperCAmelCase						)
def lowerCamelCase     (     lowerCAmelCase							:      Union[str, Any]						):





	"""simple docstring"""







	__magic_name__			:       Optional[int]	      =	tmp_path_factory.mktemp('data'						) / 'file.html'
	with pytest.raises(__UpperCAmelCase						):
		http_get('https://huggingface.co'						,						temp_file=__UpperCAmelCase						)
	with pytest.raises(__UpperCAmelCase						):
		http_head('https://huggingface.co'						)





@patch('datasets.config.HF_DATASETS_OFFLINE'						,						__UpperCAmelCase						)
def lowerCamelCase     (     lowerCAmelCase							:      Optional[int]						):





	"""simple docstring"""







	__magic_name__			:       List[Any]	      =	tmp_path_factory.mktemp('data'						) / 'file.html'
	with pytest.raises(__UpperCAmelCase						):
		ftp_get('ftp://huggingface.co'						,						temp_file=__UpperCAmelCase						)
	with pytest.raises(__UpperCAmelCase						):
		ftp_head('ftp://huggingface.co'						)





@patch('datasets.config.HF_DATASETS_OFFLINE'						,						__UpperCAmelCase						)
def lowerCamelCase     (     lowerCAmelCase							:      Tuple						):





	"""simple docstring"""







	__magic_name__			:       Tuple	      =	tmp_path_factory.mktemp('data'						) / 'file.html'
	with pytest.raises(__UpperCAmelCase						):
		fsspec_get('s3://huggingface.co'						,						temp_file=__UpperCAmelCase						)
	with pytest.raises(__UpperCAmelCase						):
		fsspec_head('s3://huggingface.co'						)




from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


a_      :      Any    =     logging.get_logger(__name__)


a_      :      str    =     TypeVar('DatasetType', Dataset, IterableDataset)






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase = None							,  __UpperCAmelCase = None							,  __UpperCAmelCase = None							,  __UpperCAmelCase = None							,  __UpperCAmelCase = "first_exhausted"							,  ):
 from .arrow_dataset import Dataset
 from .iterable_dataset import IterableDataset

 if not datasets:
  raise ValueError('''Unable to interleave an empty list of datasets.'''       )
 for i, dataset in enumerate(__UpperCAmelCase       ):
  if not isinstance(__UpperCAmelCase							,  (Dataset, IterableDataset)       ):
   if isinstance(__UpperCAmelCase							,  (DatasetDict, IterableDatasetDict)       ):
    if not dataset:
     raise ValueError(
         f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
         '''is an empty dataset dictionary.'''       )
    raise ValueError(
        f"Dataset at position {i} has at least one split: {list(__UpperCAmelCase       )}\n"
        f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__UpperCAmelCase       )       )}']"       )
   raise ValueError(
       f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__UpperCAmelCase       ).__name__}."       )
  if i == 0:
   a__							,			a__												= (
       (Dataset, IterableDataset) if isinstance(__UpperCAmelCase							,  __UpperCAmelCase       ) else (IterableDataset, Dataset)
   )
  elif not isinstance(__UpperCAmelCase							,  __UpperCAmelCase       ):
   raise ValueError(
       f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."       )
 if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
  raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy."       )
 if dataset_type is Dataset:
  return _interleave_map_style_datasets(
      __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  info=__UpperCAmelCase							,  split=__UpperCAmelCase							,  stopping_strategy=__UpperCAmelCase       )
 else:
  return _interleave_iterable_datasets(
      __UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase							,  info=__UpperCAmelCase							,  split=__UpperCAmelCase							,  stopping_strategy=__UpperCAmelCase       )






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase = None							,  __UpperCAmelCase = None							,  __UpperCAmelCase = 0							,  ):

 if not dsets:
  raise ValueError('''Unable to concatenate an empty list of datasets.'''       )
 for i, dataset in enumerate(__UpperCAmelCase       ):
  if not isinstance(__UpperCAmelCase							,  (Dataset, IterableDataset)       ):
   if isinstance(__UpperCAmelCase							,  (DatasetDict, IterableDatasetDict)       ):
    if not dataset:
     raise ValueError(
         f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
         '''is an empty dataset dictionary.'''       )
    raise ValueError(
        f"Dataset at position {i} has at least one split: {list(__UpperCAmelCase       )}\n"
        f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__UpperCAmelCase       )       )}']"       )
   raise ValueError(
       f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__UpperCAmelCase       ).__name__}."       )
  if i == 0:
   a__							,			a__												= (
       (Dataset, IterableDataset) if isinstance(__UpperCAmelCase							,  __UpperCAmelCase       ) else (IterableDataset, Dataset)
   )
  elif not isinstance(__UpperCAmelCase							,  __UpperCAmelCase       ):
   raise ValueError(
       f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."       )
 if dataset_type is Dataset:
  return _concatenate_map_style_datasets(__UpperCAmelCase							,  info=__UpperCAmelCase							,  split=__UpperCAmelCase							,  axis=__UpperCAmelCase       )
 else:
  return _concatenate_iterable_datasets(__UpperCAmelCase							,  info=__UpperCAmelCase							,  split=__UpperCAmelCase							,  axis=__UpperCAmelCase       )





