





'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available


a__			:					Optional[Any]       						=						{'tokenization_herbert': ['HerbertTokenizer']}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    a__			:					List[str]       						=						['HerbertTokenizerFast']


if TYPE_CHECKING:
    from .tokenization_herbert import HerbertTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_herbert_fast import HerbertTokenizerFast

else:
    import sys

    a__			:					List[Any]       						=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


lowerCamelCase__  =					get_logger(__name__)




class 					_UpperCAmelCase			(     enum.Enum  ):



 '''simple docstring'''




 __A			       =					'''all_checks'''
 __A			       =					'''basic_checks'''
 __A			       =					'''no_checks'''




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''






def 				lowerCAmelCase__       (			a__    ,     a__    ,     a__=None )      ->int:




  '''simple docstring'''

  if expected_checksums is None:
    logger.info("Unable to verify checksums." )
    return
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise ExpectedMoreDownloadedFiles(str(set(a__ ) - set(a__ ) ) )
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise UnexpectedDownloadedFile(str(set(a__ ) - set(a__ ) ) )
  _UpperCamelCase           =    [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
  _UpperCamelCase           =    " for " + verification_name if verification_name is not None else ""
  if len(a__ ) > 0:
    raise NonMatchingChecksumError(
        f'Checksums didn\'t match{for_verification_name}:\n'
        f'{bad_urls}\n'
        "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error" )
  logger.info("All the checksums matched successfully" + for_verification_name )




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''






def 				lowerCAmelCase__       (			a__    ,     a__ )      ->Dict:




  '''simple docstring'''

  if expected_splits is None:
    logger.info("Unable to verify splits sizes." )
    return
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise ExpectedMoreSplits(str(set(a__ ) - set(a__ ) ) )
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise UnexpectedSplits(str(set(a__ ) - set(a__ ) ) )
  _UpperCamelCase           =    [
      {"expected": expected_splits[name], "recorded": recorded_splits[name]}
      for name in expected_splits
      if expected_splits[name].num_examples != recorded_splits[name].num_examples
  ]
  if len(a__ ) > 0:
    raise NonMatchingSplitsSizesError(str(a__ ) )
  logger.info("All the splits matched successfully." )


def 				lowerCAmelCase__       (			a__    ,     a__ = True )      ->dict:




  '''simple docstring'''

  if record_checksum:
    _UpperCamelCase           =    shaaaa()
    with open(a__    ,     "rb" ) as f:
      for chunk in iter(lambda: f.read(1 << 20 )    ,     B"" ):
        m.update(a__ )
      _UpperCamelCase           =    m.hexdigest()
  else:
    _UpperCamelCase           =    None
  return {"num_bytes": os.path.getsize(a__ ), "checksum": checksum}







def 				lowerCAmelCase__       (			a__ )      ->Any:




  '''simple docstring'''

  if dataset_size and config.IN_MEMORY_MAX_SIZE:
    return dataset_size < config.IN_MEMORY_MAX_SIZE
  else:
    return False




