
import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


_A     =							logging.get_logger(__name__)

_A     =							OrderedDict(
    [
        ("audio-spectrogram-transformer", "ASTFeatureExtractor"),
        ("beit", "BeitFeatureExtractor"),
        ("chinese_clip", "ChineseCLIPFeatureExtractor"),
        ("clap", "ClapFeatureExtractor"),
        ("clip", "CLIPFeatureExtractor"),
        ("clipseg", "ViTFeatureExtractor"),
        ("conditional_detr", "ConditionalDetrFeatureExtractor"),
        ("convnext", "ConvNextFeatureExtractor"),
        ("cvt", "ConvNextFeatureExtractor"),
        ("data2vec-audio", "Wav2Vec2FeatureExtractor"),
        ("data2vec-vision", "BeitFeatureExtractor"),
        ("deformable_detr", "DeformableDetrFeatureExtractor"),
        ("deit", "DeiTFeatureExtractor"),
        ("detr", "DetrFeatureExtractor"),
        ("dinat", "ViTFeatureExtractor"),
        ("donut-swin", "DonutFeatureExtractor"),
        ("dpt", "DPTFeatureExtractor"),
        ("encodec", "EncodecFeatureExtractor"),
        ("flava", "FlavaFeatureExtractor"),
        ("glpn", "GLPNFeatureExtractor"),
        ("groupvit", "CLIPFeatureExtractor"),
        ("hubert", "Wav2Vec2FeatureExtractor"),
        ("imagegpt", "ImageGPTFeatureExtractor"),
        ("layoutlmv2", "LayoutLMv2FeatureExtractor"),
        ("layoutlmv3", "LayoutLMv3FeatureExtractor"),
        ("levit", "LevitFeatureExtractor"),
        ("maskformer", "MaskFormerFeatureExtractor"),
        ("mctct", "MCTCTFeatureExtractor"),
        ("mobilenet_v1", "MobileNetV1FeatureExtractor"),
        ("mobilenet_v2", "MobileNetV2FeatureExtractor"),
        ("mobilevit", "MobileViTFeatureExtractor"),
        ("nat", "ViTFeatureExtractor"),
        ("owlvit", "OwlViTFeatureExtractor"),
        ("perceiver", "PerceiverFeatureExtractor"),
        ("poolformer", "PoolFormerFeatureExtractor"),
        ("regnet", "ConvNextFeatureExtractor"),
        ("resnet", "ConvNextFeatureExtractor"),
        ("segformer", "SegformerFeatureExtractor"),
        ("sew", "Wav2Vec2FeatureExtractor"),
        ("sew-d", "Wav2Vec2FeatureExtractor"),
        ("speech_to_text", "Speech2TextFeatureExtractor"),
        ("speecht5", "SpeechT5FeatureExtractor"),
        ("swiftformer", "ViTFeatureExtractor"),
        ("swin", "ViTFeatureExtractor"),
        ("swinv2", "ViTFeatureExtractor"),
        ("table-transformer", "DetrFeatureExtractor"),
        ("timesformer", "VideoMAEFeatureExtractor"),
        ("tvlt", "TvltFeatureExtractor"),
        ("unispeech", "Wav2Vec2FeatureExtractor"),
        ("unispeech-sat", "Wav2Vec2FeatureExtractor"),
        ("van", "ConvNextFeatureExtractor"),
        ("videomae", "VideoMAEFeatureExtractor"),
        ("vilt", "ViltFeatureExtractor"),
        ("vit", "ViTFeatureExtractor"),
        ("vit_mae", "ViTFeatureExtractor"),
        ("vit_msn", "ViTFeatureExtractor"),
        ("wav2vec2", "Wav2Vec2FeatureExtractor"),
        ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"),
        ("wavlm", "Wav2Vec2FeatureExtractor"),
        ("whisper", "WhisperFeatureExtractor"),
        ("xclip", "CLIPFeatureExtractor"),
        ("yolos", "YolosFeatureExtractor"),
    ]
)

_A     =							_LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)






def      lowerCamelCase__       (       __lowerCAmelCase						:						int       ):




			"""simple docstring"""


			for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
						if class_name in extractors:
									lowerCAmelCase_      			=			model_type_to_module_name(lowerCAmelCase__       )

									lowerCAmelCase_      			=			importlib.import_module(F""".{module_name}"""					,							"transformers.models"       )
									try:
												return getattr(lowerCAmelCase__					,							lowerCAmelCase__       )
									except AttributeError:
												continue

			for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
						if getattr(lowerCAmelCase__					,							"__name__"					,							lowerCAmelCase__       ) == class_name:
									return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
			lowerCAmelCase_      			=			importlib.import_module("transformers"       )
			if hasattr(lowerCAmelCase__					,							lowerCAmelCase__       ):
						return getattr(lowerCAmelCase__					,							lowerCAmelCase__       )

			return None






def      lowerCamelCase__       (       __lowerCAmelCase						:						int					,							__lowerCAmelCase						:						Union[str, Any] = None					,							__lowerCAmelCase						:						Optional[int] = False					,							__lowerCAmelCase						:						Optional[Any] = False					,							__lowerCAmelCase						:						Optional[int] = None					,							__lowerCAmelCase						:						Any = None					,							__lowerCAmelCase						:						int = None					,							__lowerCAmelCase						:						int = False					,							**__lowerCAmelCase						:						List[str]					,							):




			"""simple docstring"""


			lowerCAmelCase_      			=			get_file_from_repo(
			    lowerCAmelCase__					,							lowerCAmelCase__					,							cache_dir=lowerCAmelCase__					,							force_download=lowerCAmelCase__					,							resume_download=lowerCAmelCase__					,							proxies=lowerCAmelCase__					,							use_auth_token=lowerCAmelCase__					,							revision=lowerCAmelCase__					,							local_files_only=lowerCAmelCase__					,							)
			if resolved_config_file is None:
						logger.info(
						    "Could not locate the feature extractor configuration file, will try to use the model config instead."       )
						return {}

			with open(lowerCAmelCase__					,							encoding="utf-8"       ) as reader:
						return json.load(lowerCAmelCase__       )





class 			_lowerCAmelCase      :




				def __init__(      self  )       ->   List[str]:
							raise EnvironmentError(
							    "AutoFeatureExtractor is designed to be instantiated "
							    "using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."  )




				@classmethod
				@replace_list_option_in_docstrings(snake_case__  )
				def   __a		(      cls       ,       _UpperCamelCase       ,       **_UpperCamelCase  )       ->   str:
							lowerCAmelCase_      			=			kwargs.pop("config"       ,       snake_case__  )
							lowerCAmelCase_      			=			kwargs.pop("trust_remote_code"       ,       snake_case__  )
							lowerCAmelCase_      			=			True

							lowerCAmelCase_			,						lowerCAmelCase_      			=			FeatureExtractionMixin.get_feature_extractor_dict(snake_case__       ,       **snake_case__  )
							lowerCAmelCase_      			=			config_dict.get("feature_extractor_type"       ,       snake_case__  )
							lowerCAmelCase_      			=			None
							if "AutoFeatureExtractor" in config_dict.get("auto_map"       ,       {}  ):
										lowerCAmelCase_      			=			config_dict["auto_map"]["AutoFeatureExtractor"]

							# If we don't find the feature extractor class in the feature extractor config, let's try the model config.
							if feature_extractor_class is None and feature_extractor_auto_map is None:
										if not isinstance(snake_case__       ,       snake_case__  ):
													lowerCAmelCase_      			=			AutoConfig.from_pretrained(snake_case__       ,       **snake_case__  )
										# It could be in `config.feature_extractor_type``
										lowerCAmelCase_      			=			getattr(snake_case__       ,       "feature_extractor_type"       ,       snake_case__  )
										if hasattr(snake_case__       ,       "auto_map"  ) and "AutoFeatureExtractor" in config.auto_map:
													lowerCAmelCase_      			=			config.auto_map["AutoFeatureExtractor"]

							if feature_extractor_class is not None:
										lowerCAmelCase_      			=			feature_extractor_class_from_name(snake_case__  )

							lowerCAmelCase_      			=			feature_extractor_auto_map is not None
							lowerCAmelCase_      			=			feature_extractor_class is not None or type(snake_case__  ) in FEATURE_EXTRACTOR_MAPPING
							lowerCAmelCase_      			=			resolve_trust_remote_code(
							    snake_case__       ,       snake_case__       ,       snake_case__       ,       snake_case__  )

							if has_remote_code and trust_remote_code:
										lowerCAmelCase_      			=			get_class_from_dynamic_module(
										    snake_case__       ,       snake_case__       ,       **snake_case__  )
										lowerCAmelCase_      			=			kwargs.pop("code_revision"       ,       snake_case__  )
										if os.path.isdir(snake_case__  ):
													feature_extractor_class.register_for_auto_class()
										return feature_extractor_class.from_dict(snake_case__       ,       **snake_case__  )
							elif feature_extractor_class is not None:
										return feature_extractor_class.from_dict(snake_case__       ,       **snake_case__  )
							# Last try: we use the FEATURE_EXTRACTOR_MAPPING.
							elif type(snake_case__  ) in FEATURE_EXTRACTOR_MAPPING:
										lowerCAmelCase_      			=			FEATURE_EXTRACTOR_MAPPING[type(snake_case__  )]
										return feature_extractor_class.from_dict(snake_case__       ,       **snake_case__  )

							raise ValueError(
							    f"""Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a """
							    f"""`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following """
							    f"""`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()  )}"""  )




				@staticmethod
				def   __a		(      _UpperCamelCase       ,       _UpperCamelCase  )       ->   Optional[Any]:
							FEATURE_EXTRACTOR_MAPPING.register(snake_case__       ,       snake_case__  )





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A     =  "▁"

_A     =  {"vocab_file": "spiece.model"}

_A     =  {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

_A     =  {
    "google/pegasus-xsum": 5_12,
}


_A     =  logging.get_logger(__name__)





class 			_lowerCAmelCase      (			__a ):
				_lowercase    	=VOCAB_FILES_NAMES

				_lowercase    	=VOCAB_FILES_NAMES
				_lowercase    	=PRETRAINED_VOCAB_FILES_MAP
				_lowercase    	=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowercase    	=['''input_ids''', '''attention_mask''']




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase="<pad>"       ,       _UpperCamelCase="</s>"       ,       _UpperCamelCase="<unk>"       ,       _UpperCamelCase="<mask_2>"       ,       _UpperCamelCase="<mask_1>"       ,       _UpperCamelCase=None       ,       _UpperCamelCase=103       ,       _UpperCamelCase = None       ,       **_UpperCamelCase       ,       )       ->   None:
							lowerCAmelCase_      			=			offset
							if additional_special_tokens is not None:
										if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ):
													raise TypeError(
													    f"""additional_special_tokens should be of type {type(_UpperCamelCase  )}, but is"""
													    f""" {type(_UpperCamelCase  )}"""  )

										lowerCAmelCase_      			=			(
										    ([mask_token_sent] + additional_special_tokens)
										    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
										    else additional_special_tokens
										)
										# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
										additional_special_tokens_extended += [
										    f"""<unk_{i}>""" for i in range(len(_UpperCamelCase  )       ,       self.offset - 1  )
										]

										if len(set(_UpperCamelCase  )  ) != len(_UpperCamelCase  ):
													raise ValueError(
													    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
													    f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""  )
										lowerCAmelCase_      			=			additional_special_tokens_extended
							else:
										lowerCAmelCase_      			=			[mask_token_sent] if mask_token_sent is not None else []
										additional_special_tokens += [f"""<unk_{i}>""" for i in range(2       ,       self.offset  )]

							lowerCAmelCase_      			=			{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    eos_token=_UpperCamelCase       ,       unk_token=_UpperCamelCase       ,       mask_token=_UpperCamelCase       ,       pad_token=_UpperCamelCase       ,       mask_token_sent=_UpperCamelCase       ,       offset=_UpperCamelCase       ,       additional_special_tokens=_UpperCamelCase       ,       sp_model_kwargs=self.sp_model_kwargs       ,       **_UpperCamelCase       ,       )
							lowerCAmelCase_      			=			mask_token_sent
							lowerCAmelCase_      			=			vocab_file
							lowerCAmelCase_      			=			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(_UpperCamelCase  )

							# add special tokens to encoder dict
							lowerCAmelCase_      			=			{
							    0: self.pad_token,
							    1: self.eos_token,
							}

							if self.mask_token_sent is not None:
										self.encoder.update(
										    {
										        2: self.mask_token_sent,
										        3: self.mask_token,
										    }  )

							if self.offset > 0:
										# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
										# mask_token_sent is already added to list -> so start at 1
										self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1       ,       self.offset - 1  )}  )

							lowerCAmelCase_      			=			{v: k for k, v in self.encoder.items()}




				@property
				def   __a		(      self  )       ->   int:
							return len(self.sp_model  ) + self.offset




				def   __a		(      self  )       ->   Dict[str, int]:
							lowerCAmelCase_      			=			{self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
							vocab.update(self.added_tokens_encoder  )
							return vocab




				def __getstate__(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			self.__dict__.copy()
							lowerCAmelCase_      			=			None
							return state




				def __setstate__(      self       ,       _UpperCamelCase  )       ->   Optional[int]:
							lowerCAmelCase_      			=			d

							# for backward compatibility
							if not hasattr(self       ,       "sp_model_kwargs"  ):
										lowerCAmelCase_      			=			{}

							lowerCAmelCase_      			=			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(self.vocab_file  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   List[str]:
							return self.sp_model.encode(_UpperCamelCase       ,       out_type=_UpperCamelCase  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   int:
							if token in self.decoder:
										return self.decoder[token]
							elif token in self.added_tokens_decoder:
										return self.added_tokens_decoder[token]
							lowerCAmelCase_      			=			self.sp_model.piece_to_id(_UpperCamelCase  )
							return sp_id + self.offset




				def   __a		(      self       ,       _UpperCamelCase  )       ->   str:
							if index in self.encoder:
										return self.encoder[index]
							elif index in self.added_tokens_encoder:
										return self.added_tokens_encoder[index]
							else:
										lowerCAmelCase_      			=			self.sp_model.IdToPiece(index - self.offset  )
							return token




				def   __a		(      self       ,       _UpperCamelCase  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			[]
							lowerCAmelCase_      			=			""
							for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
													out_string += self.sp_model.decode(_UpperCamelCase  ) + token
													lowerCAmelCase_      			=			[]
										else:
													current_sub_tokens.append(_UpperCamelCase  )
							out_string += self.sp_model.decode(_UpperCamelCase  )
							return out_string.strip()




				def   __a		(      self       ,       _UpperCamelCase=False  )       ->   Optional[int]:
							return 1




				def   __a		(      self       ,       _UpperCamelCase  )       ->   int:
							lowerCAmelCase_      			=			set(self.all_special_ids  )  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special

							return [1 if x in all_special_ids else 0 for x in seq]




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = None       ,       _UpperCamelCase = False  )       ->   List[int]:
							if already_has_special_tokens:
										return self._special_token_mask(_UpperCamelCase  )
							elif token_ids_a is None:
										return self._special_token_mask(_UpperCamelCase  ) + [1]
							else:
										return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase=None  )       ->   List[int]:
							if token_ids_a is None:
										return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = None  )       ->   Tuple[str]:
							if not os.path.isdir(_UpperCamelCase  ):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
										return
							lowerCAmelCase_      			=			os.path.join(
							    _UpperCamelCase       ,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

							if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
										copyfile(self.vocab_file       ,       _UpperCamelCase  )
							elif not os.path.isfile(self.vocab_file  ):
										with open(_UpperCamelCase       ,       "wb"  ) as fi:
													lowerCAmelCase_      			=			self.sp_model.serialized_model_proto()
													fi.write(_UpperCamelCase  )

							return (out_vocab_file,)



