



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__snake_case     :List[Any]			       =logging.get_logger(__name__)

__snake_case     :Union[str, Any]			       ={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__snake_case     :Tuple			       ={
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__snake_case     :str			       ={
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


__snake_case     :List[Any]			       ={
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}



class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     Optional[Any]								=     VOCAB_FILES_NAMES
				A_   :     str								=     PRETRAINED_VOCAB_FILES_MAP
				A_   :     Optional[int]								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A_   :     Tuple								=     PRETRAINED_INIT_CONFIGURATION
				A_   :     Optional[Any]								=     RoFormerTokenizer


				def __init__(							self      :  Optional[Any]						, __UpperCamelCase      :  Any=None						, __UpperCamelCase      :  List[str]=None						, __UpperCamelCase      :  Union[str, Any]=True						, __UpperCamelCase      :  Union[str, Any]="[UNK]"						, __UpperCamelCase      :  Any="[SEP]"						, __UpperCamelCase      :  List[Any]="[PAD]"						, __UpperCamelCase      :  Tuple="[CLS]"						, __UpperCamelCase      :  List[str]="[MASK]"						, __UpperCamelCase      :  int=True						, __UpperCamelCase      :  Tuple=None						, **__UpperCamelCase      :  Tuple						, )		->     Union[str, Any]:
											super().__init__(
											    __UpperCamelCase						, tokenizer_file=__UpperCamelCase						, do_lower_case=__UpperCamelCase						, unk_token=__UpperCamelCase						, sep_token=__UpperCamelCase						, pad_token=__UpperCamelCase						, cls_token=__UpperCamelCase						, mask_token=__UpperCamelCase						, tokenize_chinese_chars=__UpperCamelCase						, strip_accents=__UpperCamelCase						, **__UpperCamelCase						, )

											A      						=    json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
											if (
											    pre_tok_state.get('lowercase'						, __UpperCamelCase       ) != do_lower_case
											    or pre_tok_state.get('strip_accents'						, __UpperCamelCase       ) != strip_accents
											):
																		A      						=    getattr(__UpperCamelCase						, pre_tok_state.pop('type'       )       )
																		A      						=    do_lower_case
																		A      						=    strip_accents
																		A      						=    pre_tok_class(**__UpperCamelCase       )

											A      						=    do_lower_case


				def __getstate__(							self      :  str       )		->     Optional[int]:
											A      						=    self.__dict__.copy()
											A      						=    BertPreTokenizer()
											return state


				def __setstate__(							self      :  Dict						, __UpperCamelCase      :  Any       )		->     str:
											A      						=    d
											A      						=    self.__dict__['_tokenizer'].get_vocab()
											A      						=    PreTokenizer.custom(JiebaPreTokenizer(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  Optional[int]=None       )		->     Union[str, Any]:
											A      						=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output


				def __UpperCamelCase						(							self      :  Any						, __UpperCamelCase      :  List[int]						, __UpperCamelCase      :  Optional[List[int]] = None       )		->     List[int]:
											A      						=    [self.sep_token_id]
											A      						=    [self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[str] = None       )		->     Tuple[str]:
											A      						=    self._tokenizer.model.save(__UpperCamelCase						, name=__UpperCamelCase       )
											return tuple(__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  Optional[int]=None						, __UpperCamelCase      :  Optional[int]=None						, __UpperCamelCase      :  List[str]=False						, **__UpperCamelCase      :  Union[str, Any]						, )		->     Optional[Any]:
											A      						=    BertPreTokenizer()
											return super().save_pretrained(__UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase						, __UpperCamelCase						, **__UpperCamelCase       )
import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


SCREAMING_SNAKE_CASE			:						List[Any]				  =		get_logger(__name__)
class  A_			(							enum.Enum			):
  _SCREAMING_SNAKE_CASE     					=      """all_checks"""
  _SCREAMING_SNAKE_CASE     					=      """basic_checks"""
  _SCREAMING_SNAKE_CASE     					=      """no_checks"""
class  A_			(							a_			):
  pass
class  A_			(							a_			):
  pass
class  A_			(							a_			):
  pass
class  A_			(							a_			):
  pass



def 	__A		(      _A				,					_A				,					_A=None       ):



 """simple docstring"""


 if expected_checksums is None:
  logger.info("Unable to verify checksums."       )
  return
 if len(set(_A       ) - set(_A       )       ) > 0:
  raise ExpectedMoreDownloadedFiles(str(set(_A       ) - set(_A       )       )       )
 if len(set(_A       ) - set(_A       )       ) > 0:
  raise UnexpectedDownloadedFile(str(set(_A       ) - set(_A       )       )       )
 __a							=	[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
 __a							=	" for " + verification_name if verification_name is not None else ""
 if len(_A       ) > 0:
  raise NonMatchingChecksumError(
      f"""Checksums didn't match{for_verification_name}:\n"""
      f"""{bad_urls}\n"""
      "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error"       )
 logger.info("All the checksums matched successfully" + for_verification_name       )
class  A_			(							a_			):
  pass
class  A_			(							a_			):
  pass
class  A_			(							a_			):
  pass
class  A_			(							a_			):
  pass



def 	__A		(      _A				,					_A       ):



 """simple docstring"""


 if expected_splits is None:
  logger.info("Unable to verify splits sizes."       )
  return
 if len(set(_A       ) - set(_A       )       ) > 0:
  raise ExpectedMoreSplits(str(set(_A       ) - set(_A       )       )       )
 if len(set(_A       ) - set(_A       )       ) > 0:
  raise UnexpectedSplits(str(set(_A       ) - set(_A       )       )       )
 __a							=	[
     {"expected": expected_splits[name], "recorded": recorded_splits[name]}
     for name in expected_splits
     if expected_splits[name].num_examples != recorded_splits[name].num_examples
 ]
 if len(_A       ) > 0:
  raise NonMatchingSplitsSizesError(str(_A       )       )
 logger.info("All the splits matched successfully."       )



def 	__A		(      _A				,					_A = True       ):



 """simple docstring"""


 if record_checksum:
  __a							=	shaaaa()
  with open(_A				,					"rb"       ) as f:
   for chunk in iter(lambda: f.read(1 << 20       )				,					B""       ):
    m.update(_A       )
   __a							=	m.hexdigest()
 else:
  __a							=	None
 return {"num_bytes": os.path.getsize(_A       ), "checksum": checksum}






def 	__A		(      _A       ):



 """simple docstring"""


 if dataset_size and config.IN_MEMORY_MAX_SIZE:
  return dataset_size < config.IN_MEMORY_MAX_SIZE
 else:
  return False


