






import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


A__      :     Optional[int]       =					logging.get_logger(__name__)

A__      :     List[str]       =					OrderedDict(
    [
        ('audio-spectrogram-transformer', 'ASTFeatureExtractor'),
        ('beit', 'BeitFeatureExtractor'),
        ('chinese_clip', 'ChineseCLIPFeatureExtractor'),
        ('clap', 'ClapFeatureExtractor'),
        ('clip', 'CLIPFeatureExtractor'),
        ('clipseg', 'ViTFeatureExtractor'),
        ('conditional_detr', 'ConditionalDetrFeatureExtractor'),
        ('convnext', 'ConvNextFeatureExtractor'),
        ('cvt', 'ConvNextFeatureExtractor'),
        ('data2vec-audio', 'Wav2Vec2FeatureExtractor'),
        ('data2vec-vision', 'BeitFeatureExtractor'),
        ('deformable_detr', 'DeformableDetrFeatureExtractor'),
        ('deit', 'DeiTFeatureExtractor'),
        ('detr', 'DetrFeatureExtractor'),
        ('dinat', 'ViTFeatureExtractor'),
        ('donut-swin', 'DonutFeatureExtractor'),
        ('dpt', 'DPTFeatureExtractor'),
        ('encodec', 'EncodecFeatureExtractor'),
        ('flava', 'FlavaFeatureExtractor'),
        ('glpn', 'GLPNFeatureExtractor'),
        ('groupvit', 'CLIPFeatureExtractor'),
        ('hubert', 'Wav2Vec2FeatureExtractor'),
        ('imagegpt', 'ImageGPTFeatureExtractor'),
        ('layoutlmv2', 'LayoutLMv2FeatureExtractor'),
        ('layoutlmv3', 'LayoutLMv3FeatureExtractor'),
        ('levit', 'LevitFeatureExtractor'),
        ('maskformer', 'MaskFormerFeatureExtractor'),
        ('mctct', 'MCTCTFeatureExtractor'),
        ('mobilenet_v1', 'MobileNetV1FeatureExtractor'),
        ('mobilenet_v2', 'MobileNetV2FeatureExtractor'),
        ('mobilevit', 'MobileViTFeatureExtractor'),
        ('nat', 'ViTFeatureExtractor'),
        ('owlvit', 'OwlViTFeatureExtractor'),
        ('perceiver', 'PerceiverFeatureExtractor'),
        ('poolformer', 'PoolFormerFeatureExtractor'),
        ('regnet', 'ConvNextFeatureExtractor'),
        ('resnet', 'ConvNextFeatureExtractor'),
        ('segformer', 'SegformerFeatureExtractor'),
        ('sew', 'Wav2Vec2FeatureExtractor'),
        ('sew-d', 'Wav2Vec2FeatureExtractor'),
        ('speech_to_text', 'Speech2TextFeatureExtractor'),
        ('speecht5', 'SpeechT5FeatureExtractor'),
        ('swiftformer', 'ViTFeatureExtractor'),
        ('swin', 'ViTFeatureExtractor'),
        ('swinv2', 'ViTFeatureExtractor'),
        ('table-transformer', 'DetrFeatureExtractor'),
        ('timesformer', 'VideoMAEFeatureExtractor'),
        ('tvlt', 'TvltFeatureExtractor'),
        ('unispeech', 'Wav2Vec2FeatureExtractor'),
        ('unispeech-sat', 'Wav2Vec2FeatureExtractor'),
        ('van', 'ConvNextFeatureExtractor'),
        ('videomae', 'VideoMAEFeatureExtractor'),
        ('vilt', 'ViltFeatureExtractor'),
        ('vit', 'ViTFeatureExtractor'),
        ('vit_mae', 'ViTFeatureExtractor'),
        ('vit_msn', 'ViTFeatureExtractor'),
        ('wav2vec2', 'Wav2Vec2FeatureExtractor'),
        ('wav2vec2-conformer', 'Wav2Vec2FeatureExtractor'),
        ('wavlm', 'Wav2Vec2FeatureExtractor'),
        ('whisper', 'WhisperFeatureExtractor'),
        ('xclip', 'CLIPFeatureExtractor'),
        ('yolos', 'YolosFeatureExtractor'),
    ]
)

A__      :     Tuple       =					_LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)






def     a     (		lowerCamelCase_							):






				'''simple docstring'''
				for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
								if class_name in extractors:
												lowercase__  					=     model_type_to_module_name(lowerCamelCase_							)

												lowercase__  					=     importlib.import_module(F""".{module_name}"""						,  '''transformers.models'''							)
												try:
																return getattr(lowerCamelCase_						,  lowerCamelCase_							)
												except AttributeError:
																continue

				for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
								if getattr(lowerCamelCase_						,  '''__name__'''						,  lowerCamelCase_							) == class_name:
												return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
				lowercase__  					=     importlib.import_module('''transformers'''							)
				if hasattr(lowerCamelCase_						,  lowerCamelCase_							):
								return getattr(lowerCamelCase_						,  lowerCamelCase_							)

				return None






def     a     (		lowerCamelCase_						,  lowerCamelCase_ = None						,  lowerCamelCase_ = False						,  lowerCamelCase_ = False						,  lowerCamelCase_ = None						,  lowerCamelCase_ = None						,  lowerCamelCase_ = None						,  lowerCamelCase_ = False						,  **lowerCamelCase_						,  ):






				'''simple docstring'''
				lowercase__  					=     get_file_from_repo(
				    lowerCamelCase_						,  lowerCamelCase_						,  cache_dir=lowerCamelCase_						,  force_download=lowerCamelCase_						,  resume_download=lowerCamelCase_						,  proxies=lowerCamelCase_						,  use_auth_token=lowerCamelCase_						,  revision=lowerCamelCase_						,  local_files_only=lowerCamelCase_						,  )
				if resolved_config_file is None:
								logger.info(
								    '''Could not locate the feature extractor configuration file, will try to use the model config instead.'''							)
								return {}

				with open(lowerCamelCase_						,  encoding='''utf-8'''							) as reader:
								return json.load(lowerCamelCase_							)




class  _UpperCAmelCase   :

				"""simple docstring"""




				def __init__(	self		:       Union[str, Any]    ):

								'''simple docstring'''
								raise EnvironmentError(
								    '''AutoFeatureExtractor is designed to be instantiated '''
								    '''using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method.'''    )




				@classmethod
				@replace_list_option_in_docstrings(lowerCamelCase    )
				def 							lowercase__  (	cls		:       str,    lowerCamelCase		:       str,    **lowerCamelCase		:       Dict    ):

								'''simple docstring'''
								lowercase__  					=     kwargs.pop('''config''',    lowerCamelCase    )
								lowercase__  					=     kwargs.pop('''trust_remote_code''',    lowerCamelCase    )
								lowercase__  					=     True

								lowercase__					,		lowercase__  					=     FeatureExtractionMixin.get_feature_extractor_dict(lowerCamelCase,    **lowerCamelCase    )
								lowercase__  					=     config_dict.get('''feature_extractor_type''',    lowerCamelCase    )
								lowercase__  					=     None
								if "AutoFeatureExtractor" in config_dict.get('''auto_map''',    {}    ):
												lowercase__  					=     config_dict['''auto_map''']['''AutoFeatureExtractor''']

								# If we don't find the feature extractor class in the feature extractor config, let's try the model config.
								if feature_extractor_class is None and feature_extractor_auto_map is None:
												if not isinstance(lowerCamelCase,    lowerCamelCase    ):
																lowercase__  					=     AutoConfig.from_pretrained(lowerCamelCase,    **lowerCamelCase    )
												# It could be in `config.feature_extractor_type``
												lowercase__  					=     getattr(lowerCamelCase,    '''feature_extractor_type''',    lowerCamelCase    )
												if hasattr(lowerCamelCase,    '''auto_map'''    ) and "AutoFeatureExtractor" in config.auto_map:
																lowercase__  					=     config.auto_map['''AutoFeatureExtractor''']

								if feature_extractor_class is not None:
												lowercase__  					=     feature_extractor_class_from_name(lowerCamelCase    )

								lowercase__  					=     feature_extractor_auto_map is not None
								lowercase__  					=     feature_extractor_class is not None or type(lowerCamelCase    ) in FEATURE_EXTRACTOR_MAPPING
								lowercase__  					=     resolve_trust_remote_code(
								    lowerCamelCase,    lowerCamelCase,    lowerCamelCase,    lowerCamelCase    )

								if has_remote_code and trust_remote_code:
												lowercase__  					=     get_class_from_dynamic_module(
												    lowerCamelCase,    lowerCamelCase,    **lowerCamelCase    )
												lowercase__  					=     kwargs.pop('''code_revision''',    lowerCamelCase    )
												if os.path.isdir(lowerCamelCase    ):
																feature_extractor_class.register_for_auto_class()
												return feature_extractor_class.from_dict(lowerCamelCase,    **lowerCamelCase    )
								elif feature_extractor_class is not None:
												return feature_extractor_class.from_dict(lowerCamelCase,    **lowerCamelCase    )
								# Last try: we use the FEATURE_EXTRACTOR_MAPPING.
								elif type(lowerCamelCase    ) in FEATURE_EXTRACTOR_MAPPING:
												lowercase__  					=     FEATURE_EXTRACTOR_MAPPING[type(lowerCamelCase    )]
												return feature_extractor_class.from_dict(lowerCamelCase,    **lowerCamelCase    )

								raise ValueError(
								    F"""Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a """
								    F"""`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following """
								    F"""`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()    )}"""    )




				@staticmethod
				def 							lowercase__  (	lowerCamelCase		:       str,    lowerCamelCase		:       str    ):

								'''simple docstring'''
								FEATURE_EXTRACTOR_MAPPING.register(lowerCamelCase,    lowerCamelCase    )












import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_pegasus import PegasusTokenizer
else:
							A__      :     Optional[Any]       =					None


A__      :     str       =					logging.get_logger(__name__)


A__      :     Optional[int]       =					'▁'

A__      :     List[str]       =					{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A__      :     Dict       =					{
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

A__      :     int       =					{
    'google/pegasus-xsum': 5_12,
}
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			VOCAB_FILES_NAMES
				lowercase__											=			PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=			PegasusTokenizer
				lowercase__											=			["""input_ids""", """attention_mask"""]




				def __init__(	self		:       Dict,    lowerCamelCase		:       str=None,    lowerCamelCase		:       List[str]=None,    lowerCamelCase		:       int="<pad>",    lowerCamelCase		:       str="</s>",    lowerCamelCase		:       Optional[Any]="<unk>",    lowerCamelCase		:       Any="<mask_2>",    lowerCamelCase		:       Optional[Any]="<mask_1>",    lowerCamelCase		:       int=None,    lowerCamelCase		:       Dict=103,    **lowerCamelCase		:       List[Any],    ):

								'''simple docstring'''
								lowercase__  					=     offset

								if additional_special_tokens is not None:
												if not isinstance(lowerCamelCase,    lowerCamelCase    ):
																raise TypeError(
																    F"""additional_special_tokens should be of type {type(lowerCamelCase    )}, but is"""
																    F""" {type(lowerCamelCase    )}"""    )

												lowercase__  					=     (
												    ([mask_token_sent] + additional_special_tokens)
												    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
												    else additional_special_tokens
												)
												# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
												additional_special_tokens_extended += [
												    F"""<unk_{i}>""" for i in range(len(lowerCamelCase    ),    self.offset - 1    )
												]

												if len(set(lowerCamelCase    )    ) != len(lowerCamelCase    ):
																raise ValueError(
																    '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
																    F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""    )
												lowercase__  					=     additional_special_tokens_extended
								else:
												lowercase__  					=     [mask_token_sent] if mask_token_sent is not None else []
												additional_special_tokens += [F"""<unk_{i}>""" for i in range(2,    self.offset    )]

								super().__init__(
								    lowerCamelCase,    tokenizer_file=lowerCamelCase,    pad_token=lowerCamelCase,    eos_token=lowerCamelCase,    unk_token=lowerCamelCase,    mask_token=lowerCamelCase,    mask_token_sent=lowerCamelCase,    offset=lowerCamelCase,    additional_special_tokens=lowerCamelCase,    **lowerCamelCase,    )
								lowercase__  					=     vocab_file
								lowercase__  					=     False if not self.vocab_file else True




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       int    ):

								'''simple docstring'''
								lowercase__  					=     set(self.all_special_ids    )  # call it once instead of inside list comp
								all_special_ids.remove(self.unk_token_id    )  # <unk> is only sometimes special

								if all_special_ids != set(range(len(self.additional_special_tokens    ) + 3    )    ):
												raise ValueError(
												    '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
												    F""" {len(self.additional_special_tokens    )} additional_special_tokens, but got {all_special_ids}"""    )

								return [1 if x in all_special_ids else 0 for x in seq]




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       List,    lowerCamelCase		:       Optional[List] = None,    lowerCamelCase		:       bool = False    ):

								'''simple docstring'''
								if already_has_special_tokens:
												return self._special_token_mask(lowerCamelCase    )
								elif token_ids_a is None:
												return self._special_token_mask(lowerCamelCase    ) + [1]
								else:
												return self._special_token_mask(token_ids_a + token_ids_a    ) + [1]




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       Optional[int],    lowerCamelCase		:       Optional[Any]=None    ):

								'''simple docstring'''
								if token_ids_a is None:
												return token_ids_a + [self.eos_token_id]
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return token_ids_a + token_ids_a + [self.eos_token_id]




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       str,    lowerCamelCase		:       Optional[str] = None    ):

								'''simple docstring'''
								if not self.can_save_slow_tokenizer:
												raise ValueError(
												    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
												    '''tokenizer.'''    )

								if not os.path.isdir(lowerCamelCase    ):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
												return
								lowercase__  					=     os.path.join(
								    lowerCamelCase,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

								if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase    ):
												copyfile(self.vocab_file,    lowerCamelCase    )

								return (out_vocab_file,)




