


'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


lowercase				=logging.get_logger(__name__)


lowercase				={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all MVP models at https://huggingface.co/models?filter=mvp
lowercase				={
    'vocab_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json',
    },
    'added_tokens.json': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json',
    },
    'merges_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json',
    },
}

lowercase				={
    'RUCAIBox/mvp': 1024,
}






class    __magic_name__				(						lowerCAmelCase							):
       UpperCAmelCase		 =VOCAB_FILES_NAMES
       UpperCAmelCase		 =PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase		 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCAmelCase		 =["input_ids", "attention_mask"]
       UpperCAmelCase		 =MvpTokenizer






       def __init__(     self    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case="replace"    ,				snake_case="<s>"    ,				snake_case="</s>"    ,				snake_case="</s>"    ,				snake_case="<s>"    ,				snake_case="<unk>"    ,				snake_case="<pad>"    ,				snake_case="<mask>"    ,				snake_case=False    ,				snake_case=True    ,				**snake_case    ,				)		->    Optional[Any]:




          '''simple docstring'''
          super().__init__(
              snake_case    ,				snake_case    ,				tokenizer_file=snake_case    ,				errors=snake_case    ,				bos_token=snake_case    ,				eos_token=snake_case    ,				sep_token=snake_case    ,				cls_token=snake_case    ,				unk_token=snake_case    ,				pad_token=snake_case    ,				mask_token=snake_case    ,				add_prefix_space=snake_case    ,				trim_offsets=snake_case    ,				**snake_case    ,				)

          _UpperCAmelCase  :  Tuple				       =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
          if pre_tok_state.get('add_prefix_space'    ,				snake_case) != add_prefix_space:
             _UpperCAmelCase  :  List[str]				       =getattr(snake_case    ,				pre_tok_state.pop('type'))
             _UpperCAmelCase  :  List[str]				       =add_prefix_space
             _UpperCAmelCase  :  Tuple				       =pre_tok_class(**snake_case)

          _UpperCAmelCase  :  Optional[int]				       =add_prefix_space

          # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
          _UpperCAmelCase  :  int				       ='post_processor'
          _UpperCAmelCase  :  List[str]				       =getattr(self.backend_tokenizer    ,				snake_case    ,				snake_case)
          if tokenizer_component_instance:
             _UpperCAmelCase  :  Optional[int]				       =json.loads(tokenizer_component_instance.__getstate__())

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                _UpperCAmelCase  :  Optional[Any]				       =tuple(state['sep'])
             if "cls" in state:
                _UpperCAmelCase  :  int				       =tuple(state['cls'])

             _UpperCAmelCase  :  Tuple				       =False

             if state.get('add_prefix_space'    ,				snake_case) != add_prefix_space:
                _UpperCAmelCase  :  Optional[Any]				       =add_prefix_space
                _UpperCAmelCase  :  str				       =True

             if state.get('trim_offsets'    ,				snake_case) != trim_offsets:
                _UpperCAmelCase  :  List[Any]				       =trim_offsets
                _UpperCAmelCase  :  Any				       =True

             if changes_to_apply:
                _UpperCAmelCase  :  Tuple				       =getattr(snake_case    ,				state.pop('type'))
                _UpperCAmelCase  :  List[Any]				       =component_class(**snake_case)
                setattr(self.backend_tokenizer    ,				snake_case    ,				snake_case)






       @property
       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          if self._mask_token is None:
             if self.verbose:
                logger.error('Using mask_token, but it is not set yet.')
             return None
          return str(self._mask_token)






       @mask_token.setter
       def      lowerCAmelCase						(     self    ,				snake_case)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =AddedToken(snake_case    ,				lstrip=snake_case    ,				rstrip=snake_case) if isinstance(snake_case    ,				snake_case) else value
          _UpperCAmelCase  :  List[Any]				       =value






       def      lowerCAmelCase						(     self    ,				*snake_case    ,				**snake_case)		->    BatchEncoding:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =kwargs.get('is_split_into_words'    ,				snake_case)

          if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                 'to use it with pretokenized inputs.')

          return super()._batch_encode_plus(*snake_case    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				*snake_case    ,				**snake_case)		->    BatchEncoding:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =kwargs.get('is_split_into_words'    ,				snake_case)

          if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                 'to use it with pretokenized inputs.')

          return super()._encode_plus(*snake_case    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None)		->    Tuple[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  str				       =self._tokenizer.model.save(snake_case    ,				name=snake_case)
          return tuple(snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case=None)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       =[self.bos_token_id] + token_ids_a + [self.eos_token_id]
          if token_ids_a is None:
             return output

          return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None)		->    List[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =[self.sep_token_id]
          _UpperCAmelCase  :  List[str]				       =[self.cls_token_id]

          if token_ids_a is None:
             return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





'''simple docstring'''






import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text





def lowerCamelCase__     (		__lowerCamelCase			:				List[str]				):







   '''simple docstring'''




   _UpperCAmelCase  :  int				       =[]
   for line in lines:
      _UpperCAmelCase  :  Dict				       =re.sub(R'#.*'			,       ''			,       __lowerCamelCase				)  # remove comments
      if line:
         filtered_lines.append(__lowerCamelCase				)
   _UpperCAmelCase  :  str				       ='\n'.join(__lowerCamelCase				)

   # Make a hash from all this code
   _UpperCAmelCase  :  str				       =full_str.encode('utf-8'				)
   return shaaaa(__lowerCamelCase				).hexdigest()


# get importable module names and hash for caching
lowercase				={
    'csv': (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    'json': (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    'pandas': (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    'parquet': (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    'arrow': (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    'text': (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    'imagefolder': (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    'audiofolder': (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
lowercase				={
    '.csv': ('csv', {}),
    '.tsv': ('csv', {'sep': '\t'}),
    '.json': ('json', {}),
    '.jsonl': ('json', {}),
    '.parquet': ('parquet', {}),
    '.arrow': ('arrow', {}),
    '.txt': ('text', {}),
}
_EXTENSION_TO_MODULE.update({ext: ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
lowercase				={'imagefolder', 'audiofolder'}

# Used to filter data files based on extensions given a module name
lowercase				={}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
 _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('.zip')




_MODULE_TO_EXTENSIONS["audiofolder"].append('.zip')

