
import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text

def 		UpperCamelCase						(   lowercase_ )		->						str:


  '''simple docstring'''






  lowercase__	: Optional[Any]     		=   []
  for line in lines:
    lowercase__	: Optional[int]     		=   re.sub(R"""#.*"""			,			""""""			,			lowerCAmelCase_ )  # remove comments
    if line:
      filtered_lines.append(lowerCAmelCase_ )
  lowercase__	: List[Any]     		=   """\n""".join(lowerCAmelCase_ )

  # Make a hash from all this code
  lowercase__	: Tuple     		=   full_str.encode("""utf-8""" )
  return shaaaa(lowerCAmelCase_ ).hexdigest()


# get importable module names and hash for caching
lowerCamelCase__     :		Dict        = {
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
lowerCamelCase__     :		Dict        = {
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
lowerCamelCase__     :		Optional[int]        = {"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
lowerCamelCase__     :		Optional[Any]        = {}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
     _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append(""".zip""")



_MODULE_TO_EXTENSIONS["audiofolder"].append(""".zip""")











"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


snake_case   					=				logging.get_logger(__name__)

snake_case   					=				{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case   					=				{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

snake_case   					=				{
    '''junnyu/roformer_chinese_small''': 1_5_3_6,
    '''junnyu/roformer_chinese_base''': 1_5_3_6,
    '''junnyu/roformer_chinese_char_small''': 5_1_2,
    '''junnyu/roformer_chinese_char_base''': 5_1_2,
    '''junnyu/roformer_small_discriminator''': 1_2_8,
    '''junnyu/roformer_small_generator''': 1_2_8,
}


snake_case   					=				{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	str           =  VOCAB_FILES_NAMES
   A__		:	Tuple           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	List[Any]           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Dict           =  PRETRAINED_INIT_CONFIGURATION
   A__		:	Any           =  RoFormerTokenizer



   def __init__(       self    :	Optional[int]							,       __lowerCamelCase    :	Any=None							,       __lowerCamelCase    :	Optional[Any]=None							,       __lowerCamelCase    :	int=True							,       __lowerCamelCase    :	Any="[UNK]"							,       __lowerCamelCase    :	Tuple="[SEP]"							,       __lowerCamelCase    :	Optional[Any]="[PAD]"							,       __lowerCamelCase    :	Union[str, Any]="[CLS]"							,       __lowerCamelCase    :	int="[MASK]"							,       __lowerCamelCase    :	str=True							,       __lowerCamelCase    :	List[Any]=None							,       **__lowerCamelCase    :	Any							,       ):



     """simple docstring"""
     super().__init__(
         __lowerCamelCase							,       tokenizer_file=__lowerCamelCase							,       do_lower_case=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       sep_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       cls_token=__lowerCamelCase							,       mask_token=__lowerCamelCase							,       tokenize_chinese_chars=__lowerCamelCase							,       strip_accents=__lowerCamelCase							,       **__lowerCamelCase							,       )

     _snake_case   	=      json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
     if (
         pre_tok_state.get('''lowercase'''							,       __lowerCamelCase				) != do_lower_case
         or pre_tok_state.get('''strip_accents'''							,       __lowerCamelCase				) != strip_accents
     ):
       _snake_case   	=      getattr(__lowerCamelCase							,       pre_tok_state.pop('''type'''				)				)
       _snake_case   	=      do_lower_case
       _snake_case   	=      strip_accents
       _snake_case   	=      pre_tok_class(**__lowerCamelCase				)

     _snake_case   	=      do_lower_case



   def __getstate__(       self    :	int				):



     """simple docstring"""
     _snake_case   	=      self.__dict__.copy()
     _snake_case   	=      BertPreTokenizer()
     return state



   def __setstate__(       self    :	Dict							,       __lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      d
     _snake_case   	=      self.__dict__['''_tokenizer'''].get_vocab()
     _snake_case   	=      PreTokenizer.custom(JiebaPreTokenizer(__lowerCamelCase				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	List[Any]=None				):



     """simple docstring"""
     _snake_case   	=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     _snake_case   	=      [self.sep_token_id]
     _snake_case   	=      [self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
     return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     _snake_case   	=      self._tokenizer.model.save(__lowerCamelCase							,       name=__lowerCamelCase				)
     return tuple(__lowerCamelCase				)





   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	List[str]=None							,       __lowerCamelCase    :	Union[str, Any]=None							,       __lowerCamelCase    :	List[str]=False							,       **__lowerCamelCase    :	List[Any]							,       ):



     """simple docstring"""
     _snake_case   	=      BertPreTokenizer()
     return super().save_pretrained(__lowerCamelCase							,       __lowerCamelCase							,       __lowerCamelCase							,       __lowerCamelCase							,       **__lowerCamelCase				)





