



"""simple docstring"""






import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case   					=				logging.get_logger(__name__)

snake_case   					=				'''▁'''

snake_case   					=				{'''vocab_file''': '''prophetnet.tokenizer'''}

snake_case   					=				{
    '''vocab_file''': {
        '''microsoft/xprophetnet-large-wiki100-cased''': (
            '''https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'''
        ),
    }
}

snake_case   					=				{
    '''microsoft/xprophetnet-large-wiki100-cased''': {'''do_lower_case''': False},
}

snake_case   					=				{
    '''microsoft/xprophetnet-large-wiki100-cased''': 5_1_2,
}




def 			snake_case		(		lowerCAmelCase_	)						->				List[str]:
  _snake_case   	=      collections.OrderedDict()
  with open(lowerCAmelCase_  ,						'''r'''  ,						encoding='''utf-8'''	) as reader:
    _snake_case   	=      reader.readlines()
  for index, token in enumerate(lowerCAmelCase_	):
    _snake_case   	=      token.rstrip('''\n'''	)
    _snake_case   	=      index
  return vocab







class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	Optional[int]           =  VOCAB_FILES_NAMES
   A__		:	Tuple           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	int           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Optional[int]           =  ['''input_ids''', '''attention_mask''']



   def __init__(       self    :	Optional[Any]							,       __lowerCamelCase    :	Optional[Any]							,       __lowerCamelCase    :	Any="[SEP]"							,       __lowerCamelCase    :	List[Any]="[SEP]"							,       __lowerCamelCase    :	Optional[Any]="[SEP]"							,       __lowerCamelCase    :	Any="[UNK]"							,       __lowerCamelCase    :	str="[PAD]"							,       __lowerCamelCase    :	Union[str, Any]="[CLS]"							,       __lowerCamelCase    :	Tuple="[MASK]"							,       __lowerCamelCase    :	Optional[Dict[str, Any]] = None							,       **__lowerCamelCase    :	List[Any]							,       ):



     """simple docstring"""
     _snake_case   	=      {} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       sep_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       cls_token=__lowerCamelCase							,       mask_token=__lowerCamelCase							,       sp_model_kwargs=self.sp_model_kwargs							,       **__lowerCamelCase							,       )

     try:
       import sentencepiece as spm
     except ImportError:
       logger.warning(
           '''You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'''
           ''' pip install sentencepiece'''				)
       raise

     _snake_case   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(str(__lowerCamelCase				)				)
     _snake_case   	=      vocab_file

     # Original fairseq vocab and spm vocab must be "aligned":
     # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
     # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
     # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
     # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

     # put special tokens and [unused] tokens into the vocab
     _snake_case   	=      {'''[PAD]''': 0, '''[CLS]''': 1, '''[SEP]''': 2, '''[UNK]''': 3, '''[MASK]''': 4}

     for i in range(1_0				):
       _snake_case   	=      f"""[unused{i}]"""
       _snake_case   	=      5 + i

     # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
     _snake_case   	=      1_2
     _snake_case   	=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}
     for k in self.fairseq_tokens_to_ids.keys():
       self.unique_no_split_tokens.append(__lowerCamelCase				)



   def __getstate__(       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      self.__dict__.copy()
     _snake_case   	=      None
     return state



   def __setstate__(       self    :	Optional[int]							,       __lowerCamelCase    :	Any				):



     """simple docstring"""
     _snake_case   	=      d
     try:
       import sentencepiece as spm
     except ImportError:
       logger.warning(
           '''You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'''
           ''' pip install sentencepiece'''				)
       raise

     # for backward compatibility
     if not hasattr(self							,       '''sp_model_kwargs'''				):
       _snake_case   	=      {}

     _snake_case   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(self.vocab_file				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase							,       token_ids_a=__lowerCamelCase							,       already_has_special_tokens=__lowerCamelCase				)

     if token_ids_a is None:
       return ([0] * len(__lowerCamelCase				)) + [1]
     return ([0] * len(__lowerCamelCase				)) + [1] + ([0] * len(__lowerCamelCase				)) + [1]



   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     _snake_case   	=      [self.sep_token_id]

     if token_ids_a is None:
       return len(token_ids_a + sep				) * [0]
     return len(token_ids_a + sep + sep + token_ids_a + sep				) * [0]



   @property
   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     return len(self.sp_model				) + self.fairseq_offset



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      {self.convert_ids_to_tokens(__lowerCamelCase				): i for i in range(self.vocab_size				)}
     vocab.update(self.added_tokens_encoder				)
     return vocab



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     return self.sp_model.encode(__lowerCamelCase							,       out_type=__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
     _snake_case   	=      self.sp_model.PieceToId(__lowerCamelCase				)

     # Need to return unknown token if the SP model returned 0
     return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	Tuple				):



     """simple docstring"""
     if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
     return self.sp_model.IdToPiece(index - self.fairseq_offset				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      ''''''.join(__lowerCamelCase				).replace(__lowerCamelCase							,       ''' '''				).strip()
     return out_string



   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if not os.path.isdir(__lowerCamelCase				):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
     _snake_case   	=      os.path.join(
         __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(__lowerCamelCase				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file							,       __lowerCamelCase				)
     elif not os.path.isfile(self.vocab_file				):
       with open(__lowerCamelCase							,       '''wb'''				) as fi:
         _snake_case   	=      self.sp_model.serialized_model_proto()
         fi.write(__lowerCamelCase				)

     return (out_vocab_file,)





   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""

     if token_ids_a is None:
       return token_ids_a + [self.sep_token_id]
     _snake_case   	=      [self.sep_token_id]
     return token_ids_a + sep + token_ids_a + sep










"""simple docstring"""






import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all image processors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...image_processing_utils import ImageProcessingMixin
from ...utils import CONFIG_NAME, IMAGE_PROCESSOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


snake_case   					=				logging.get_logger(__name__)

snake_case   					=				OrderedDict(
    [
        ('''align''', '''EfficientNetImageProcessor'''),
        ('''beit''', '''BeitImageProcessor'''),
        ('''bit''', '''BitImageProcessor'''),
        ('''blip''', '''BlipImageProcessor'''),
        ('''blip-2''', '''BlipImageProcessor'''),
        ('''bridgetower''', '''BridgeTowerImageProcessor'''),
        ('''chinese_clip''', '''ChineseCLIPImageProcessor'''),
        ('''clip''', '''CLIPImageProcessor'''),
        ('''clipseg''', '''ViTImageProcessor'''),
        ('''conditional_detr''', '''ConditionalDetrImageProcessor'''),
        ('''convnext''', '''ConvNextImageProcessor'''),
        ('''convnextv2''', '''ConvNextImageProcessor'''),
        ('''cvt''', '''ConvNextImageProcessor'''),
        ('''data2vec-vision''', '''BeitImageProcessor'''),
        ('''deformable_detr''', '''DeformableDetrImageProcessor'''),
        ('''deit''', '''DeiTImageProcessor'''),
        ('''deta''', '''DetaImageProcessor'''),
        ('''detr''', '''DetrImageProcessor'''),
        ('''dinat''', '''ViTImageProcessor'''),
        ('''donut-swin''', '''DonutImageProcessor'''),
        ('''dpt''', '''DPTImageProcessor'''),
        ('''efficientformer''', '''EfficientFormerImageProcessor'''),
        ('''efficientnet''', '''EfficientNetImageProcessor'''),
        ('''flava''', '''FlavaImageProcessor'''),
        ('''focalnet''', '''BitImageProcessor'''),
        ('''git''', '''CLIPImageProcessor'''),
        ('''glpn''', '''GLPNImageProcessor'''),
        ('''groupvit''', '''CLIPImageProcessor'''),
        ('''imagegpt''', '''ImageGPTImageProcessor'''),
        ('''instructblip''', '''BlipImageProcessor'''),
        ('''layoutlmv2''', '''LayoutLMv2ImageProcessor'''),
        ('''layoutlmv3''', '''LayoutLMv3ImageProcessor'''),
        ('''levit''', '''LevitImageProcessor'''),
        ('''mask2former''', '''Mask2FormerImageProcessor'''),
        ('''maskformer''', '''MaskFormerImageProcessor'''),
        ('''mgp-str''', '''ViTImageProcessor'''),
        ('''mobilenet_v1''', '''MobileNetV1ImageProcessor'''),
        ('''mobilenet_v2''', '''MobileNetV2ImageProcessor'''),
        ('''mobilevit''', '''MobileViTImageProcessor'''),
        ('''mobilevit''', '''MobileViTImageProcessor'''),
        ('''mobilevitv2''', '''MobileViTImageProcessor'''),
        ('''nat''', '''ViTImageProcessor'''),
        ('''oneformer''', '''OneFormerImageProcessor'''),
        ('''owlvit''', '''OwlViTImageProcessor'''),
        ('''perceiver''', '''PerceiverImageProcessor'''),
        ('''pix2struct''', '''Pix2StructImageProcessor'''),
        ('''poolformer''', '''PoolFormerImageProcessor'''),
        ('''regnet''', '''ConvNextImageProcessor'''),
        ('''resnet''', '''ConvNextImageProcessor'''),
        ('''sam''', '''SamImageProcessor'''),
        ('''segformer''', '''SegformerImageProcessor'''),
        ('''swiftformer''', '''ViTImageProcessor'''),
        ('''swin''', '''ViTImageProcessor'''),
        ('''swin2sr''', '''Swin2SRImageProcessor'''),
        ('''swinv2''', '''ViTImageProcessor'''),
        ('''table-transformer''', '''DetrImageProcessor'''),
        ('''timesformer''', '''VideoMAEImageProcessor'''),
        ('''tvlt''', '''TvltImageProcessor'''),
        ('''upernet''', '''SegformerImageProcessor'''),
        ('''van''', '''ConvNextImageProcessor'''),
        ('''videomae''', '''VideoMAEImageProcessor'''),
        ('''vilt''', '''ViltImageProcessor'''),
        ('''vit''', '''ViTImageProcessor'''),
        ('''vit_hybrid''', '''ViTHybridImageProcessor'''),
        ('''vit_mae''', '''ViTImageProcessor'''),
        ('''vit_msn''', '''ViTImageProcessor'''),
        ('''xclip''', '''CLIPImageProcessor'''),
        ('''yolos''', '''YolosImageProcessor'''),
    ]
)

snake_case   					=				_LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)




def 			snake_case		(		lowerCAmelCase_	)						->				str:
  for module_name, extractors in IMAGE_PROCESSOR_MAPPING_NAMES.items():
    if class_name in extractors:
      _snake_case   	=      model_type_to_module_name(lowerCAmelCase_	)

      _snake_case   	=      importlib.import_module(f""".{module_name}"""  ,						'''transformers.models'''	)
      try:
        return getattr(lowerCAmelCase_  ,						lowerCAmelCase_	)
      except AttributeError:
        continue

  for _, extractor in IMAGE_PROCESSOR_MAPPING._extra_content.items():
    if getattr(lowerCAmelCase_  ,						'''__name__'''  ,						lowerCAmelCase_	) == class_name:
      return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  _snake_case   	=      importlib.import_module('''transformers'''	)
  if hasattr(lowerCAmelCase_  ,						lowerCAmelCase_	):
    return getattr(lowerCAmelCase_  ,						lowerCAmelCase_	)

  return None




def 			snake_case		(		lowerCAmelCase_  ,						lowerCAmelCase_ = None  ,						lowerCAmelCase_ = False  ,						lowerCAmelCase_ = False  ,						lowerCAmelCase_ = None  ,						lowerCAmelCase_ = None  ,						lowerCAmelCase_ = None  ,						lowerCAmelCase_ = False  ,						**lowerCAmelCase_  ,						)						->				int:
  _snake_case   	=      get_file_from_repo(
      lowerCAmelCase_  ,						lowerCAmelCase_  ,						cache_dir=lowerCAmelCase_  ,						force_download=lowerCAmelCase_  ,						resume_download=lowerCAmelCase_  ,						proxies=lowerCAmelCase_  ,						use_auth_token=lowerCAmelCase_  ,						revision=lowerCAmelCase_  ,						local_files_only=lowerCAmelCase_  ,						)
  if resolved_config_file is None:
    logger.info(
        '''Could not locate the image processor configuration file, will try to use the model config instead.'''	)
    return {}

  with open(lowerCAmelCase_  ,						encoding='''utf-8'''	) as reader:
    return json.load(lowerCAmelCase_	)







class     UpperCAmelCase    :



   def __init__(       self    :	Any				):



     """simple docstring"""
     raise EnvironmentError(
         '''AutoImageProcessor is designed to be instantiated '''
         '''using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method.'''				)



   @classmethod
   @replace_list_option_in_docstrings(__lowerCamelCase				)
   def    __UpperCAmelCase    (       cls    :	List[str]							,       __lowerCamelCase    :	List[str]							,       **__lowerCamelCase    :	Dict				):



     """simple docstring"""
     _snake_case   	=      kwargs.pop('''config'''							,       __lowerCamelCase				)
     _snake_case   	=      kwargs.pop('''trust_remote_code'''							,       __lowerCamelCase				)
     _snake_case   	=      True

     _snake_case	,     _snake_case   	=      ImageProcessingMixin.get_image_processor_dict(__lowerCamelCase							,       **__lowerCamelCase				)
     _snake_case   	=      config_dict.get('''image_processor_type'''							,       __lowerCamelCase				)
     _snake_case   	=      None
     if "AutoImageProcessor" in config_dict.get('''auto_map'''							,       {}				):
       _snake_case   	=      config_dict['''auto_map''']['''AutoImageProcessor''']

     # If we still don't have the image processor class, check if we're loading from a previous feature extractor config
     # and if so, infer the image processor class from there.
     if image_processor_class is None and image_processor_auto_map is None:
       _snake_case   	=      config_dict.pop('''feature_extractor_type'''							,       __lowerCamelCase				)
       if feature_extractor_class is not None:
         logger.warning(
             '''Could not find image processor class in the image processor config or the model config. Loading'''
             ''' based on pattern matching with the model\'s feature extractor configuration.'''				)
         _snake_case   	=      feature_extractor_class.replace('''FeatureExtractor'''							,       '''ImageProcessor'''				)
       if "AutoFeatureExtractor" in config_dict.get('''auto_map'''							,       {}				):
         _snake_case   	=      config_dict['''auto_map''']['''AutoFeatureExtractor''']
         _snake_case   	=      feature_extractor_auto_map.replace('''FeatureExtractor'''							,       '''ImageProcessor'''				)
         logger.warning(
             '''Could not find image processor auto map in the image processor config or the model config.'''
             ''' Loading based on pattern matching with the model\'s feature extractor configuration.'''				)

        # If we don't find the image processor class in the image processor config, let's try the model config.
     if image_processor_class is None and image_processor_auto_map is None:
       if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         _snake_case   	=      AutoConfig.from_pretrained(__lowerCamelCase							,       **__lowerCamelCase				)
       # It could be in `config.image_processor_type``
       _snake_case   	=      getattr(__lowerCamelCase							,       '''image_processor_type'''							,       __lowerCamelCase				)
       if hasattr(__lowerCamelCase							,       '''auto_map'''				) and "AutoImageProcessor" in config.auto_map:
         _snake_case   	=      config.auto_map['''AutoImageProcessor''']

     if image_processor_class is not None:
       _snake_case   	=      image_processor_class_from_name(__lowerCamelCase				)

     _snake_case   	=      image_processor_auto_map is not None
     _snake_case   	=      image_processor_class is not None or type(__lowerCamelCase				) in IMAGE_PROCESSOR_MAPPING
     _snake_case   	=      resolve_trust_remote_code(
         __lowerCamelCase							,       __lowerCamelCase							,       __lowerCamelCase							,       __lowerCamelCase				)

     if has_remote_code and trust_remote_code:
       _snake_case   	=      get_class_from_dynamic_module(
           __lowerCamelCase							,       __lowerCamelCase							,       **__lowerCamelCase				)
       _snake_case   	=      kwargs.pop('''code_revision'''							,       __lowerCamelCase				)
       if os.path.isdir(__lowerCamelCase				):
         image_processor_class.register_for_auto_class()
       return image_processor_class.from_dict(__lowerCamelCase							,       **__lowerCamelCase				)
     elif image_processor_class is not None:
       return image_processor_class.from_dict(__lowerCamelCase							,       **__lowerCamelCase				)
     # Last try: we use the IMAGE_PROCESSOR_MAPPING.
     elif type(__lowerCamelCase				) in IMAGE_PROCESSOR_MAPPING:
       _snake_case   	=      IMAGE_PROCESSOR_MAPPING[type(__lowerCamelCase				)]
       return image_processor_class.from_dict(__lowerCamelCase							,       **__lowerCamelCase				)

     raise ValueError(
         f"""Unrecognized image processor in {pretrained_model_name_or_path}. Should have a """
         f"""`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following """
         f"""`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES.keys()				)}"""				)





   @staticmethod
   def    __UpperCAmelCase    (       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     IMAGE_PROCESSOR_MAPPING.register(__lowerCamelCase							,       __lowerCamelCase				)





