

"""simple docstring"""




import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


__UpperCAmelCase =				'\\n    Text data.\n    Second line of data.'

__UpperCAmelCase =				'file'
@pytest.fixture(scope="""session"""      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  List[str]					   =					tmp_path_factory.mktemp("""data"""      ) / (FILE_PATH + """.zstd""")
 UpperCAmelCase__					:  Any					   =					bytes(__UpperCamelCase							,    """utf-8"""      )
 with zstd.open(__UpperCamelCase							,    """wb"""      ) as f:
  f.write(__UpperCamelCase      )
 return path
@pytest.fixture
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 with open(os.path.join(tmpfs.local_root_dir							,    __UpperCamelCase      )							,    """w"""      ) as f:
  f.write(__UpperCamelCase      )
 return FILE_PATH
@pytest.mark.parametrize("""compression_format"""							,    ["""gzip""", """xz""", """zstd"""]      )
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Dict					   =					{"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_path}
 UpperCAmelCase__					:  Optional[int]					   =					input_paths[compression_format]
 UpperCAmelCase__					:  Union[str, Any]					   =					tmp_path / """cache"""
 UpperCAmelCase__					:  List[Any]					   =					DownloadConfig(cache_dir=__UpperCamelCase							,    extract_compressed_file=__UpperCamelCase      )
 UpperCAmelCase__					:  Dict					   =					cached_path(__UpperCamelCase							,    download_config=__UpperCamelCase      )
 with open(__UpperCamelCase      ) as f:
  UpperCAmelCase__					:  Union[str, Any]					   =					f.read()
 with open(__UpperCamelCase      ) as f:
  UpperCAmelCase__					:  Tuple					   =					f.read()
 assert extracted_file_content == expected_file_content
@pytest.mark.parametrize("""default_extracted"""							,    [True, False]      )
@pytest.mark.parametrize("""default_cache_dir"""							,    [True, False]      )
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Optional[int]					   =					"""custom_cache"""
 UpperCAmelCase__					:  Any					   =					"""custom_extracted_dir"""
 UpperCAmelCase__					:  Dict					   =					tmp_path / """custom_extracted_path"""
 if default_extracted:
  UpperCAmelCase__					:  Optional[int]					   =					("""downloads""" if default_cache_dir else custom_cache_dir, """extracted""")
 else:
  monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_DIR"""							,    __UpperCamelCase      )
  monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""							,    str(__UpperCamelCase      )      )
  UpperCAmelCase__					:  Optional[Any]					   =					custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

 UpperCAmelCase__					:  Tuple					   =					xz_file
 UpperCAmelCase__					:  List[Any]					   =					(
     DownloadConfig(extract_compressed_file=__UpperCamelCase      )
     if default_cache_dir
     else DownloadConfig(cache_dir=tmp_path / custom_cache_dir							,    extract_compressed_file=__UpperCamelCase      )
 )
 UpperCAmelCase__					:  Union[str, Any]					   =					cached_path(__UpperCamelCase							,    download_config=__UpperCamelCase      )
 assert Path(__UpperCamelCase      ).parent.parts[-2:] == expected
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Union[str, Any]					   =					str(Path(__UpperCamelCase      ).resolve()      )
 assert cached_path(__UpperCamelCase      ) == text_file
 # relative path
 UpperCAmelCase__					:  int					   =					str(Path(__UpperCamelCase      ).resolve().relative_to(Path(os.getcwd()      )      )      )
 assert cached_path(__UpperCamelCase      ) == text_file
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  List[Any]					   =					str(tmp_path.resolve() / """__missing_file__.txt"""      )
 with pytest.raises(__UpperCamelCase      ):
  cached_path(__UpperCamelCase      )
 # relative path
 UpperCAmelCase__					:  str					   =					"""./__missing_file__.txt"""
 with pytest.raises(__UpperCamelCase      ):
  cached_path(__UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  str					   =					get_from_cache(F"tmp://{tmpfs_file}"      )
 with open(__UpperCamelCase      ) as f:
  UpperCAmelCase__					:  List[str]					   =					f.read()
 assert output_file_content == FILE_CONTENT
@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					):
 '''simple docstring'''





 with pytest.raises(__UpperCamelCase      ):
  cached_path("""https://huggingface.co"""      )
@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Any					   =					tmp_path_factory.mktemp("""data"""      ) / """file.html"""
 with pytest.raises(__UpperCamelCase      ):
  http_get("""https://huggingface.co"""							,    temp_file=__UpperCamelCase      )
 with pytest.raises(__UpperCamelCase      ):
  http_head("""https://huggingface.co"""      )
@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  str					   =					tmp_path_factory.mktemp("""data"""      ) / """file.html"""
 with pytest.raises(__UpperCamelCase      ):
  ftp_get("""ftp://huggingface.co"""							,    temp_file=__UpperCamelCase      )
 with pytest.raises(__UpperCamelCase      ):
  ftp_head("""ftp://huggingface.co"""      )




@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Dict					   =					tmp_path_factory.mktemp("""data"""      ) / """file.html"""
 with pytest.raises(__UpperCamelCase      ):
  fsspec_get("""s3://huggingface.co"""							,    temp_file=__UpperCamelCase      )
 with pytest.raises(__UpperCamelCase      ):
  fsspec_head("""s3://huggingface.co"""      )








"""simple docstring"""




from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


__UpperCAmelCase =				logging.get_logger(__name__)


__UpperCAmelCase =				TypeVar('DatasetType', Dataset, IterableDataset)
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase = None							,    __UpperCamelCase = None							,    __UpperCamelCase = None							,    __UpperCamelCase = None							,    __UpperCamelCase = "first_exhausted"							,    ):
 '''simple docstring'''





 from .arrow_dataset import Dataset
 from .iterable_dataset import IterableDataset

 if not datasets:
  raise ValueError("""Unable to interleave an empty list of datasets."""      )
 for i, dataset in enumerate(__UpperCamelCase      ):
  if not isinstance(__UpperCamelCase							,    (Dataset, IterableDataset)      ):
   if isinstance(__UpperCamelCase							,    (DatasetDict, IterableDatasetDict)      ):
    if not dataset:
     raise ValueError(
         F"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
         """is an empty dataset dictionary."""      )
    raise ValueError(
        F"Dataset at position {i} has at least one split: {list(__UpperCamelCase      )}\n"
        F"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__UpperCamelCase      )      )}']"      )
   raise ValueError(
       F"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__UpperCamelCase      ).__name__}."      )
  if i == 0:
   UpperCAmelCase__   ,      UpperCAmelCase__					:  List[str]					   =					(
       (Dataset, IterableDataset) if isinstance(__UpperCamelCase							,    __UpperCamelCase      ) else (IterableDataset, Dataset)
   )
  elif not isinstance(__UpperCamelCase							,    __UpperCamelCase      ):
   raise ValueError(
       F"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."      )
 if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
  raise ValueError(F"{stopping_strategy} is not supported. Please enter a valid stopping_strategy."      )
 if dataset_type is Dataset:
  return _interleave_map_style_datasets(
      __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    info=__UpperCamelCase							,    split=__UpperCamelCase							,    stopping_strategy=__UpperCamelCase      )
 else:
  return _interleave_iterable_datasets(
      __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    info=__UpperCamelCase							,    split=__UpperCamelCase							,    stopping_strategy=__UpperCamelCase      )




def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase = None							,    __UpperCamelCase = None							,    __UpperCamelCase = 0							,    ):
 '''simple docstring'''





 if not dsets:
  raise ValueError("""Unable to concatenate an empty list of datasets."""      )
 for i, dataset in enumerate(__UpperCamelCase      ):
  if not isinstance(__UpperCamelCase							,    (Dataset, IterableDataset)      ):
   if isinstance(__UpperCamelCase							,    (DatasetDict, IterableDatasetDict)      ):
    if not dataset:
     raise ValueError(
         F"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
         """is an empty dataset dictionary."""      )
    raise ValueError(
        F"Dataset at position {i} has at least one split: {list(__UpperCamelCase      )}\n"
        F"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__UpperCamelCase      )      )}']"      )
   raise ValueError(
       F"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__UpperCamelCase      ).__name__}."      )
  if i == 0:
   UpperCAmelCase__   ,      UpperCAmelCase__					:  Optional[int]					   =					(
       (Dataset, IterableDataset) if isinstance(__UpperCamelCase							,    __UpperCamelCase      ) else (IterableDataset, Dataset)
   )
  elif not isinstance(__UpperCamelCase							,    __UpperCamelCase      ):
   raise ValueError(
       F"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."      )
 if dataset_type is Dataset:
  return _concatenate_map_style_datasets(__UpperCamelCase							,    info=__UpperCamelCase							,    split=__UpperCamelCase							,    axis=__UpperCamelCase      )
 else:
  return _concatenate_iterable_datasets(__UpperCamelCase							,    info=__UpperCamelCase							,    split=__UpperCamelCase							,    axis=__UpperCamelCase      )





