




"""simple docstring"""

import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


a         =      '''http://www.mocksite.com/file1.txt'''
a         =      '''"text": ["foo", "foo"]'''
a         =      '''6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'''





class     lowercase_	:



 '''simple docstring'''
 UpperCAmelCase   :					List[Any]				  =     200
 UpperCAmelCase   :					List[str]				  =     {'''Content-Length''': '''100'''}
 UpperCAmelCase   :					str				  =     {}

 def       lowerCAmelCase_							(			self	:	List[str]   ,							**_UpperCAmelCase	:	Union[str, Any]		):
       return [bytes(_UpperCAmelCase   ,							'utf-8'		)]






def 						_snake_case					( *_snake_case							: str    ,  **_snake_case							: str							)   -> Tuple:






      '''simple docstring'''


      return MockResponse()






@pytest.mark.parametrize('urls_type'    ,  [str, list, dict]							)
def 						_snake_case					( _snake_case							: Any    ,  _snake_case							: int    ,  _snake_case							: int							)   -> Dict:






      '''simple docstring'''


      import requests

      monkeypatch.setattr(_snake_case    ,  'request'    ,  _snake_case							)

      _A        =       URL
      if issubclass(_snake_case    ,  _snake_case							):
            _A        =       url
      elif issubclass(_snake_case    ,  _snake_case							):
            _A        =       [url]
      elif issubclass(_snake_case    ,  _snake_case							):
            _A        =       {'train': url}
      _A        =       'dummy'
      _A        =       'downloads'
      _A        =       tmp_path
      _A        =       DownloadConfig(
          cache_dir=os.path.join(_snake_case    ,  _snake_case							)    ,  use_etag=_snake_case    ,  )
      _A        =       DownloadManager(dataset_name=_snake_case    ,  download_config=_snake_case							)
      _A        =       dl_manager.download(_snake_case							)
      _A        =       urls
      for downloaded_paths in [downloaded_paths]:
            if isinstance(_snake_case    ,  _snake_case							):
                  _A        =       [downloaded_paths]
                  _A        =       [urls]
            elif isinstance(_snake_case    ,  _snake_case							):
                  assert "train" in downloaded_paths.keys()
                  _A        =       downloaded_paths.values()
                  _A        =       urls.values()
            assert downloaded_paths
            for downloaded_path, input_url in zip(_snake_case    ,  _snake_case							):
                  assert downloaded_path == dl_manager.downloaded_paths[input_url]
                  _A        =       Path(_snake_case							)
                  _A        =       downloaded_path.parts
                  assert parts[-1] == HASH
                  assert parts[-2] == cache_subdir
                  assert downloaded_path.exists()
                  _A        =       downloaded_path.read_text()
                  assert content == CONTENT
                  _A        =       downloaded_path.with_suffix('.json'							)
                  assert metadata_downloaded_path.exists()
                  _A        =       json.loads(metadata_downloaded_path.read_text()							)
                  assert metadata_content == {"url": URL, "etag": None}






@pytest.mark.parametrize('paths_type'    ,  [str, list, dict]							)
def 						_snake_case					( _snake_case							: Dict    ,  _snake_case							: int    ,  _snake_case							: List[Any]							)   -> Any:






      '''simple docstring'''


      _A        =       str(_snake_case							)
      if issubclass(_snake_case    ,  _snake_case							):
            _A        =       filename
      elif issubclass(_snake_case    ,  _snake_case							):
            _A        =       [filename]
      elif issubclass(_snake_case    ,  _snake_case							):
            _A        =       {'train': filename}
      _A        =       'dummy'
      _A        =       xz_file.parent
      _A        =       'extracted'
      _A        =       DownloadConfig(
          cache_dir=_snake_case    ,  use_etag=_snake_case    ,  )
      _A        =       DownloadManager(dataset_name=_snake_case    ,  download_config=_snake_case							)
      _A        =       dl_manager.extract(_snake_case							)
      _A        =       paths
      for extracted_paths in [extracted_paths]:
            if isinstance(_snake_case    ,  _snake_case							):
                  _A        =       [extracted_paths]
                  _A        =       [paths]
            elif isinstance(_snake_case    ,  _snake_case							):
                  assert "train" in extracted_paths.keys()
                  _A        =       extracted_paths.values()
                  _A        =       paths.values()
            assert extracted_paths
            for extracted_path, input_path in zip(_snake_case    ,  _snake_case							):
                  assert extracted_path == dl_manager.extracted_paths[input_path]
                  _A        =       Path(_snake_case							)
                  _A        =       extracted_path.parts
                  assert parts[-1] == hash_url_to_filename(_snake_case    ,  etag=_snake_case							)
                  assert parts[-2] == extracted_subdir
                  assert extracted_path.exists()
                  _A        =       extracted_path.read_text()
                  _A        =       text_file.read_text()
                  assert extracted_file_content == expected_file_content






def 						_snake_case					( _snake_case							: List[str]    ,  _snake_case							: int							)   -> Any:






      '''simple docstring'''


      assert path.endswith('.jsonl'							)
      for num_items, line in enumerate(_snake_case    ,  start=1							):
            _A        =       json.loads(line.decode('utf-8'							)							)
            assert item.keys() == {"col_1", "col_2", "col_3"}
      assert num_items == 4






@pytest.mark.parametrize('archive_jsonl'    ,  ['tar_jsonl_path', 'zip_jsonl_path']							)
def 						_snake_case					( _snake_case							: int    ,  _snake_case							: int							)   -> Dict:






      '''simple docstring'''


      _A        =       request.getfixturevalue(_snake_case							)
      _A        =       DownloadManager()
      for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(_snake_case							)    ,  start=1							):
            _test_jsonl(_snake_case    ,  _snake_case							)
      assert num_jsonl == 2






@pytest.mark.parametrize('archive_nested_jsonl'    ,  ['tar_nested_jsonl_path', 'zip_nested_jsonl_path']							)
def 						_snake_case					( _snake_case							: Tuple    ,  _snake_case							: Union[str, Any]							)   -> List[Any]:






      '''simple docstring'''


      _A        =       request.getfixturevalue(_snake_case							)
      _A        =       DownloadManager()
      for num_tar, (path, file) in enumerate(dl_manager.iter_archive(_snake_case							)    ,  start=1							):
            for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(_snake_case							)    ,  start=1							):
                  _test_jsonl(_snake_case    ,  _snake_case							)
      assert num_tar == 1
      assert num_jsonl == 2







def 						_snake_case					( _snake_case							: Union[str, Any]							)   -> Any:






      '''simple docstring'''


      _A        =       DownloadManager()
      for num_file, file in enumerate(dl_manager.iter_files(_snake_case							)    ,  start=1							):
            assert os.path.basename(_snake_case							) == ("test.txt" if num_file == 1 else "train.txt")
      assert num_file == 2







"""simple docstring"""

import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


a         =      get_logger(__name__)





class     lowercase_	(					enum.Enum							):



 '''simple docstring'''
 UpperCAmelCase   :					Optional[int]				  =     '''all_checks'''
 UpperCAmelCase   :					List[Any]				  =     '''basic_checks'''
 UpperCAmelCase   :					Any				  =     '''no_checks'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''






def 						_snake_case					( _snake_case							: Optional[dict]    ,  _snake_case							: dict    ,  _snake_case							: Dict=None							)   -> Dict:






      '''simple docstring'''


      if expected_checksums is None:
            logger.info('Unable to verify checksums.'							)
            return
      if len(set(_snake_case							) - set(_snake_case							)							) > 0:
            raise ExpectedMoreDownloadedFiles(str(set(_snake_case							) - set(_snake_case							)							)							)
      if len(set(_snake_case							) - set(_snake_case							)							) > 0:
            raise UnexpectedDownloadedFile(str(set(_snake_case							) - set(_snake_case							)							)							)
      _A        =       [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
      _A        =       ' for ' + verification_name if verification_name is not None else ''
      if len(_snake_case							) > 0:
            raise NonMatchingChecksumError(
                F'''Checksums didn\'t match{for_verification_name}:\n'''
                F'''{bad_urls}\n'''
                'Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error'							)
      logger.info('All the checksums matched successfully' + for_verification_name							)





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''






def 						_snake_case					( _snake_case							: Optional[dict]    ,  _snake_case							: dict							)   -> List[str]:






      '''simple docstring'''


      if expected_splits is None:
            logger.info('Unable to verify splits sizes.'							)
            return
      if len(set(_snake_case							) - set(_snake_case							)							) > 0:
            raise ExpectedMoreSplits(str(set(_snake_case							) - set(_snake_case							)							)							)
      if len(set(_snake_case							) - set(_snake_case							)							) > 0:
            raise UnexpectedSplits(str(set(_snake_case							) - set(_snake_case							)							)							)
      _A        =       [
          {'expected': expected_splits[name], 'recorded': recorded_splits[name]}
          for name in expected_splits
          if expected_splits[name].num_examples != recorded_splits[name].num_examples
      ]
      if len(_snake_case							) > 0:
            raise NonMatchingSplitsSizesError(str(_snake_case							)							)
      logger.info('All the splits matched successfully.'							)






def 						_snake_case					( _snake_case							: str    ,  _snake_case							: bool = True							)   -> dict:






      '''simple docstring'''


      if record_checksum:
            _A        =       shaaaa()
            with open(_snake_case    ,  'rb'							) as f:
                  for chunk in iter(lambda: f.read(1 << 20							)    ,  B''							):
                        m.update(_snake_case							)
                  _A        =       m.hexdigest()
      else:
            _A        =       None
      return {"num_bytes": os.path.getsize(_snake_case							), "checksum": checksum}







def 						_snake_case					( _snake_case							: int							)   -> int:






      '''simple docstring'''


      if dataset_size and config.IN_MEMORY_MAX_SIZE:
            return dataset_size < config.IN_MEMORY_MAX_SIZE
      else:
            return False

