"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a							=logging.get_logger(__name__)


a							={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all MVP models at https://huggingface.co/models?filter=mvp
a							={
    'vocab_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json',
    },
    'added_tokens.json': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json',
    },
    'merges_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt',
    },
}

a							={
    'RUCAIBox/mvp': 1024,
}





@lru_cache()
def 			lowerCamelCase_ (					)							->					Optional[Any]:




						'''simple docstring'''



						lowerCamelCase__					=(
						    list(range(ord("!"				)   ,    ord("~"				) + 1				)				) + list(range(ord("¡"				)   ,    ord("¬"				) + 1				)				) + list(range(ord("®"				)   ,    ord("ÿ"				) + 1				)				)
						)
						lowerCamelCase__					=bs[:]
						lowerCamelCase__					=0
						for b in range(2**8				):
												if b not in bs:
																		bs.append(__lowerCAmelCase				)
																		cs.append(2**8 + n				)
																		n += 1
						lowerCamelCase__					=[chr(__lowerCAmelCase				) for n in cs]
						return dict(zip(__lowerCAmelCase   ,    __lowerCAmelCase				)				)





def 			lowerCamelCase_ (					__lowerCAmelCase				)							->					Dict:




						'''simple docstring'''



						lowerCamelCase__					=set()
						lowerCamelCase__					=word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												lowerCamelCase__					=char
						return pairs






class    __UpperCAmelCase						(	__lowerCAmelCase	):
				A__      :   Tuple  							=       VOCAB_FILES_NAMES
				A__      :   Optional[int]  							=       PRETRAINED_VOCAB_FILES_MAP
				A__      :   Dict  							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A__      :   List[str]  							=       ['''input_ids''', '''attention_mask''']

				def __init__(      self	, _lowerCamelCase	, _lowerCamelCase	, _lowerCamelCase="replace"	, _lowerCamelCase="<s>"	, _lowerCamelCase="</s>"	, _lowerCamelCase="</s>"	, _lowerCamelCase="<s>"	, _lowerCamelCase="<unk>"	, _lowerCamelCase="<pad>"	, _lowerCamelCase="<mask>"	, _lowerCamelCase=False	, **_lowerCamelCase	, ):
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else bos_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else eos_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else sep_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else cls_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else unk_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else mask_token

										super().__init__(
										    errors=_lowerCamelCase	, bos_token=_lowerCamelCase	, eos_token=_lowerCamelCase	, unk_token=_lowerCamelCase	, sep_token=_lowerCamelCase	, cls_token=_lowerCamelCase	, pad_token=_lowerCamelCase	, mask_token=_lowerCamelCase	, add_prefix_space=_lowerCamelCase	, **_lowerCamelCase	, )
										with open(_lowerCamelCase	, encoding="utf-8"      ) as vocab_handle:
																lowerCamelCase__					=json.load(_lowerCamelCase      )
										lowerCamelCase__					={v: k for k, v in self.encoder.items()}
										lowerCamelCase__					=errors  # how to handle errors in decoding
										lowerCamelCase__					=bytes_to_unicode()
										lowerCamelCase__					={v: k for k, v in self.byte_encoder.items()}
										with open(_lowerCamelCase	, encoding="utf-8"      ) as merges_handle:
																lowerCamelCase__					=merges_handle.read().split("\n"      )[1:-1]
										lowerCamelCase__					=[tuple(merge.split()      ) for merge in bpe_merges]
										lowerCamelCase__					=dict(zip(_lowerCamelCase	, range(len(_lowerCamelCase      )      )      )      )
										lowerCamelCase__					={}
										lowerCamelCase__					=add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										lowerCamelCase__					=re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"      )

				@property
				def 		_a	(      self      ):
										return len(self.encoder      )

				def 		_a	(      self      ):
										return dict(self.encoder	, **self.added_tokens_encoder      )

				def 		_a	(      self	, _lowerCamelCase      ):
										if token in self.cache:
																return self.cache[token]
										lowerCamelCase__					=tuple(_lowerCamelCase      )
										lowerCamelCase__					=get_pairs(_lowerCamelCase      )

										if not pairs:
																return token

										while True:
																lowerCamelCase__					=min(_lowerCamelCase	, key=lambda _lowerCamelCase      : self.bpe_ranks.get(_lowerCamelCase	, float("inf"      )      )      )
																if bigram not in self.bpe_ranks:
																						break
																lowerCamelCase__					,      lowerCamelCase__					=bigram
																lowerCamelCase__					=[]
																lowerCamelCase__					=0
																while i < len(_lowerCamelCase      ):
																						try:
																												lowerCamelCase__					=word.index(_lowerCamelCase	, _lowerCamelCase      )
																						except ValueError:
																												new_word.extend(word[i:]      )
																												break
																						else:
																												new_word.extend(word[i:j]      )
																												lowerCamelCase__					=j

																						if word[i] == first and i < len(_lowerCamelCase      ) - 1 and word[i + 1] == second:
																												new_word.append(first + second      )
																												i += 2
																						else:
																												new_word.append(word[i]      )
																												i += 1
																lowerCamelCase__					=tuple(_lowerCamelCase      )
																lowerCamelCase__					=new_word
																if len(_lowerCamelCase      ) == 1:
																						break
																else:
																						lowerCamelCase__					=get_pairs(_lowerCamelCase      )
										lowerCamelCase__					=" ".join(_lowerCamelCase      )
										lowerCamelCase__					=word
										return word

				def 		_a	(      self	, _lowerCamelCase      ):
										lowerCamelCase__					=[]
										for token in re.findall(self.pat	, _lowerCamelCase      ):
																lowerCamelCase__					="".join(
																    self.byte_encoder[b] for b in token.encode("utf-8"      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase      ).split(" "      )      )
										return bpe_tokens

				def 		_a	(      self	, _lowerCamelCase      ):
										return self.encoder.get(_lowerCamelCase	, self.encoder.get(self.unk_token      )      )

				def 		_a	(      self	, _lowerCamelCase      ):
										return self.decoder.get(_lowerCamelCase      )

				def 		_a	(      self	, _lowerCamelCase      ):
										lowerCamelCase__					="".join(_lowerCamelCase      )
										lowerCamelCase__					=bytearray([self.byte_decoder[c] for c in text]      ).decode("utf-8"	, errors=self.errors      )
										return text

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase = None      ):
										if not os.path.isdir(_lowerCamelCase      ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
																return
										lowerCamelCase__					=os.path.join(
										    _lowerCamelCase	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
										lowerCamelCase__					=os.path.join(
										    _lowerCamelCase	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

										with open(_lowerCamelCase	, "w"	, encoding="utf-8"      ) as f:
																f.write(json.dumps(self.encoder	, indent=2	, sort_keys=_lowerCamelCase	, ensure_ascii=_lowerCamelCase      ) + "\n"      )

										lowerCamelCase__					=0
										with open(_lowerCamelCase	, "w"	, encoding="utf-8"      ) as writer:
																writer.write("#version: 0.2\n"      )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	, key=lambda _lowerCamelCase      : kv[1]      ):
																						if index != token_index:
																												logger.warning(
																												    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																												    " Please check that the tokenizer is not corrupted!"      )
																												lowerCamelCase__					=token_index
																						writer.write(" ".join(_lowerCamelCase      ) + "\n"      )
																						index += 1

										return vocab_file, merge_file

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase = None      ):
										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										lowerCamelCase__					=[self.cls_token_id]
										lowerCamelCase__					=[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase = None	, _lowerCamelCase = False      ):
										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCamelCase	, token_ids_a=_lowerCamelCase	, already_has_special_tokens=_lowerCamelCase      )

										if token_ids_a is None:
																return [1] + ([0] * len(_lowerCamelCase      )) + [1]
										return [1] + ([0] * len(_lowerCamelCase      )) + [1, 1] + ([0] * len(_lowerCamelCase      )) + [1]

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase = None      ):
										lowerCamelCase__					=[self.sep_token_id]
										lowerCamelCase__					=[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep      ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]






				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase=False	, **_lowerCamelCase      ):
										lowerCamelCase__					=kwargs.pop("add_prefix_space"	, self.add_prefix_space      )
										if (is_split_into_words or add_prefix_space) and (len(_lowerCamelCase      ) > 0 and not text[0].isspace()):
																lowerCamelCase__					=" " + text
										return (text, kwargs)









import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


lowerCAmelCase_        =			logging.get_logger(__name__)

lowerCAmelCase_        =			OrderedDict(
    [
        ("""align""", """AlignProcessor"""),
        ("""altclip""", """AltCLIPProcessor"""),
        ("""bark""", """BarkProcessor"""),
        ("""blip""", """BlipProcessor"""),
        ("""blip-2""", """Blip2Processor"""),
        ("""bridgetower""", """BridgeTowerProcessor"""),
        ("""chinese_clip""", """ChineseCLIPProcessor"""),
        ("""clap""", """ClapProcessor"""),
        ("""clip""", """CLIPProcessor"""),
        ("""clipseg""", """CLIPSegProcessor"""),
        ("""flava""", """FlavaProcessor"""),
        ("""git""", """GitProcessor"""),
        ("""groupvit""", """CLIPProcessor"""),
        ("""hubert""", """Wav2Vec2Processor"""),
        ("""instructblip""", """InstructBlipProcessor"""),
        ("""layoutlmv2""", """LayoutLMv2Processor"""),
        ("""layoutlmv3""", """LayoutLMv3Processor"""),
        ("""markuplm""", """MarkupLMProcessor"""),
        ("""mctct""", """MCTCTProcessor"""),
        ("""mgp-str""", """MgpstrProcessor"""),
        ("""oneformer""", """OneFormerProcessor"""),
        ("""owlvit""", """OwlViTProcessor"""),
        ("""pix2struct""", """Pix2StructProcessor"""),
        ("""sam""", """SamProcessor"""),
        ("""sew""", """Wav2Vec2Processor"""),
        ("""sew-d""", """Wav2Vec2Processor"""),
        ("""speech_to_text""", """Speech2TextProcessor"""),
        ("""speech_to_text_2""", """Speech2Text2Processor"""),
        ("""speecht5""", """SpeechT5Processor"""),
        ("""trocr""", """TrOCRProcessor"""),
        ("""tvlt""", """TvltProcessor"""),
        ("""unispeech""", """Wav2Vec2Processor"""),
        ("""unispeech-sat""", """Wav2Vec2Processor"""),
        ("""vilt""", """ViltProcessor"""),
        ("""vision-text-dual-encoder""", """VisionTextDualEncoderProcessor"""),
        ("""wav2vec2""", """Wav2Vec2Processor"""),
        ("""wav2vec2-conformer""", """Wav2Vec2Processor"""),
        ("""wavlm""", """Wav2Vec2Processor"""),
        ("""whisper""", """WhisperProcessor"""),
        ("""xclip""", """XCLIPProcessor"""),
    ]
)

lowerCAmelCase_        =			_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)


def 			lowerCamelCase_		(					lowerCAmelCase:		str )->						Tuple:
  for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
    if class_name in processors:
      _snake_case      : Optional[Any]														=		model_type_to_module_name(lowerCAmelCase )

      _snake_case      : Tuple														=		importlib.import_module(F""".{module_name}"""			,							'transformers.models' )
      try:
        return getattr(lowerCAmelCase			,							lowerCAmelCase )
      except AttributeError:
        continue

  for processor in PROCESSOR_MAPPING._extra_content.values():
    if getattr(lowerCAmelCase			,							'__name__'			,							lowerCAmelCase ) == class_name:
      return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  _snake_case      : Union[str, Any]														=		importlib.import_module('transformers' )
  if hasattr(lowerCAmelCase			,							lowerCAmelCase ):
    return getattr(lowerCAmelCase			,							lowerCAmelCase )

  return None







class 				_lowerCAmelCase			:


  '''simple docstring'''







  def __init__(					self					:    Optional[Any]							):


    '''simple docstring'''





    raise EnvironmentError(
        'AutoProcessor is designed to be instantiated '
        'using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method.'							)







  @classmethod
  @replace_list_option_in_docstrings(UpperCamelCase							)
  def UpperCamelCase_       (					cls					:    Optional[Any]					, UpperCamelCase					:    Optional[Any]					, **UpperCamelCase					:    Dict							):


    '''simple docstring'''





    _snake_case      : Optional[int]														=		kwargs.pop('config'					, UpperCamelCase							)
    _snake_case      : str														=		kwargs.pop('trust_remote_code'					, UpperCamelCase							)
    _snake_case      : List[Any]														=		True

    _snake_case      : str														=		None
    _snake_case      : str														=		None

    # First, let's see if we have a preprocessor config.
    # Filter the kwargs for `get_file_from_repo`.
    _snake_case      : Dict														=		{
        key: kwargs[key] for key in inspect.signature(UpperCamelCase							).parameters.keys() if key in kwargs
    }
    # Let's start by checking whether the processor class is saved in an image processor
    _snake_case      : List[Any]														=		get_file_from_repo(
        UpperCamelCase					, UpperCamelCase					, **UpperCamelCase							)
    if preprocessor_config_file is not None:
      _snake_case							,    _snake_case      : Optional[int]														=		ImageProcessingMixin.get_image_processor_dict(UpperCamelCase					, **UpperCamelCase							)
      _snake_case      : List[str]														=		config_dict.get('processor_class'					, UpperCamelCase							)
      if "AutoProcessor" in config_dict.get('auto_map'					, {}							):
        _snake_case      : Optional[Any]														=		config_dict['auto_map']['AutoProcessor']

        # If not found, let's check whether the processor class is saved in a feature extractor config
    if preprocessor_config_file is not None and processor_class is None:
      _snake_case							,    _snake_case      : Union[str, Any]														=		FeatureExtractionMixin.get_feature_extractor_dict(UpperCamelCase					, **UpperCamelCase							)
      _snake_case      : int														=		config_dict.get('processor_class'					, UpperCamelCase							)
      if "AutoProcessor" in config_dict.get('auto_map'					, {}							):
        _snake_case      : str														=		config_dict['auto_map']['AutoProcessor']

    if processor_class is None:
      # Next, let's check whether the processor class is saved in a tokenizer
      _snake_case      : Union[str, Any]														=		get_file_from_repo(
          UpperCamelCase					, UpperCamelCase					, **UpperCamelCase							)
      if tokenizer_config_file is not None:
        with open(UpperCamelCase					, encoding='utf-8'							) as reader:
          _snake_case      : Optional[int]														=		json.load(UpperCamelCase							)

        _snake_case      : int														=		config_dict.get('processor_class'					, UpperCamelCase							)
        if "AutoProcessor" in config_dict.get('auto_map'					, {}							):
          _snake_case      : str														=		config_dict['auto_map']['AutoProcessor']

    if processor_class is None:
      # Otherwise, load config, if it can be loaded.
      if not isinstance(UpperCamelCase					, UpperCamelCase							):
        _snake_case      : Dict														=		AutoConfig.from_pretrained(
            UpperCamelCase					, trust_remote_code=UpperCamelCase					, **UpperCamelCase							)

      # And check if the config contains the processor class.
      _snake_case      : str														=		getattr(UpperCamelCase					, 'processor_class'					, UpperCamelCase							)
      if hasattr(UpperCamelCase					, 'auto_map'							) and "AutoProcessor" in config.auto_map:
        _snake_case      : Any														=		config.auto_map['AutoProcessor']

    if processor_class is not None:
      _snake_case      : Tuple														=		processor_class_from_name(UpperCamelCase							)

    _snake_case      : int														=		processor_auto_map is not None
    _snake_case      : Optional[int]														=		processor_class is not None or type(UpperCamelCase							) in PROCESSOR_MAPPING
    _snake_case      : Union[str, Any]														=		resolve_trust_remote_code(
        UpperCamelCase					, UpperCamelCase					, UpperCamelCase					, UpperCamelCase							)

    if has_remote_code and trust_remote_code:
      _snake_case      : List[str]														=		get_class_from_dynamic_module(
          UpperCamelCase					, UpperCamelCase					, **UpperCamelCase							)
      _snake_case      : Optional[Any]														=		kwargs.pop('code_revision'					, UpperCamelCase							)
      if os.path.isdir(UpperCamelCase							):
        processor_class.register_for_auto_class()
      return processor_class.from_pretrained(
          UpperCamelCase					, trust_remote_code=UpperCamelCase					, **UpperCamelCase							)
    elif processor_class is not None:
      return processor_class.from_pretrained(
          UpperCamelCase					, trust_remote_code=UpperCamelCase					, **UpperCamelCase							)
    # Last try: we use the PROCESSOR_MAPPING.
    elif type(UpperCamelCase							) in PROCESSOR_MAPPING:
      return PROCESSOR_MAPPING[type(UpperCamelCase							)].from_pretrained(UpperCamelCase					, **UpperCamelCase							)

    # At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
    # tokenizer.
    try:
      return AutoTokenizer.from_pretrained(
          UpperCamelCase					, trust_remote_code=UpperCamelCase					, **UpperCamelCase							)
    except Exception:
      try:
        return AutoImageProcessor.from_pretrained(
            UpperCamelCase					, trust_remote_code=UpperCamelCase					, **UpperCamelCase							)
      except Exception:
        pass

      try:
        return AutoFeatureExtractor.from_pretrained(
            UpperCamelCase					, trust_remote_code=UpperCamelCase					, **UpperCamelCase							)
      except Exception:
        pass

    raise ValueError(
        f"""Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a """
        'tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains'
        'the files of at least one of those processing classes.'							)







  @staticmethod
  def UpperCamelCase_       (					UpperCamelCase					:    int					, UpperCamelCase					:    str							):


    '''simple docstring'''





    PROCESSOR_MAPPING.register(UpperCamelCase					, UpperCamelCase							)




