






'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__A	:	int     =     logging.get_logger(__name__)


__A	:	Union[str, Any]     =     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
__A	:	Union[str, Any]     =     {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

__A	:	str     =     {
    'facebook/bart-base': 1_024,
    'facebook/bart-large': 1_024,
    'facebook/bart-large-mnli': 1_024,
    'facebook/bart-large-cnn': 1_024,
    'facebook/bart-large-xsum': 1_024,
    'yjernite/bart_eli5': 1_024,
}
class 						__UpperCamelCase	(						UpperCamelCase__							):
							lowercase			:		Dict   		=  VOCAB_FILES_NAMES
							lowercase			:		Any   		=  PRETRAINED_VOCAB_FILES_MAP
							lowercase			:		Optional[int]   		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase			:		Dict   		=  ['input_ids', 'attention_mask']
							lowercase			:		int   		=  BartTokenizer





							def __init__(  self		:str  ,_UpperCamelCase		:Optional[Any]=None  ,_UpperCamelCase		:List[Any]=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:int="replace"  ,_UpperCamelCase		:str="<s>"  ,_UpperCamelCase		:Tuple="</s>"  ,_UpperCamelCase		:Dict="</s>"  ,_UpperCamelCase		:Union[str, Any]="<s>"  ,_UpperCamelCase		:int="<unk>"  ,_UpperCamelCase		:Tuple="<pad>"  ,_UpperCamelCase		:Union[str, Any]="<mask>"  ,_UpperCamelCase		:Any=False  ,_UpperCamelCase		:Optional[int]=True  ,**_UpperCamelCase		:List[str]  ,):
									super().__init__(
									    __A  ,__A  ,tokenizer_file=__A  ,errors=__A  ,bos_token=__A  ,eos_token=__A  ,sep_token=__A  ,cls_token=__A  ,unk_token=__A  ,pad_token=__A  ,mask_token=__A  ,add_prefix_space=__A  ,trim_offsets=__A  ,**__A  ,)

									snake_case_		:							int			       =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
									if pre_tok_state.get("""add_prefix_space"""  ,__A			) != add_prefix_space:
											snake_case_		:							int			       =						getattr(__A  ,pre_tok_state.pop("""type"""			)			)
											snake_case_		:							Optional[int]			       =						add_prefix_space
											snake_case_		:							int			       =						pre_tok_class(**__A			)

									snake_case_		:							str			       =						add_prefix_space

									# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
									snake_case_		:							Optional[int]			       =						"""post_processor"""
									snake_case_		:							List[str]			       =						getattr(self.backend_tokenizer  ,__A  ,__A			)
									if tokenizer_component_instance:
											snake_case_		:							List[str]			       =						json.loads(tokenizer_component_instance.__getstate__()			)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
													snake_case_		:							Any			       =						tuple(state["""sep"""]			)
											if "cls" in state:
													snake_case_		:							Tuple			       =						tuple(state["""cls"""]			)

											snake_case_		:							Dict			       =						False

											if state.get("""add_prefix_space"""  ,__A			) != add_prefix_space:
													snake_case_		:							Union[str, Any]			       =						add_prefix_space
													snake_case_		:							Optional[int]			       =						True

											if state.get("""trim_offsets"""  ,__A			) != trim_offsets:
													snake_case_		:							List[Any]			       =						trim_offsets
													snake_case_		:							int			       =						True

											if changes_to_apply:
													snake_case_		:							List[str]			       =						getattr(__A  ,state.pop("""type"""			)			)
													snake_case_		:							Tuple			       =						component_class(**__A			)
													setattr(self.backend_tokenizer  ,__A  ,__A			)





							@property
							def 		a__   (  self		:int			):



									if self._mask_token is None:
											if self.verbose:
													logger.error("""Using mask_token, but it is not set yet."""			)
											return None
									return str(self._mask_token			)





							@mask_token.setter
							def 		a__   (  self		:List[Any]  ,_UpperCamelCase		:List[str]			):
									snake_case_		:							Optional[Any]			       =						AddedToken(__A  ,lstrip=__A  ,rstrip=__A			) if isinstance(__A  ,__A			) else value
									snake_case_		:							Union[str, Any]			       =						value





							def 		a__   (  self		:Optional[Any]  ,*_UpperCamelCase		:Any  ,**_UpperCamelCase		:Tuple			):
									snake_case_		:							Dict			       =						kwargs.get("""is_split_into_words"""  ,__A			)

									if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    """to use it with pretokenized inputs."""			)

									return super()._batch_encode_plus(*__A  ,**__A			)





							def 		a__   (  self		:str  ,*_UpperCamelCase		:List[Any]  ,**_UpperCamelCase		:str			):
									snake_case_		:							List[str]			       =						kwargs.get("""is_split_into_words"""  ,__A			)

									if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    """to use it with pretokenized inputs."""			)

									return super()._encode_plus(*__A  ,**__A			)





							def 		a__   (  self		:Optional[int]  ,_UpperCamelCase		:str  ,_UpperCamelCase		:Optional[str] = None			):
									snake_case_		:							Union[str, Any]			       =						self._tokenizer.model.save(__A  ,name=__A			)
									return tuple(__A			)





							def 		a__   (  self		:List[Any]  ,_UpperCamelCase		:Dict  ,_UpperCamelCase		:int=None			):
									snake_case_		:							Any			       =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
											return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





							def 		a__   (  self		:Any  ,_UpperCamelCase		:List[int]  ,_UpperCamelCase		:Optional[List[int]] = None			):
									snake_case_		:							str			       =						[self.sep_token_id]
									snake_case_		:							Union[str, Any]			       =						[self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep			) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]



'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_speech_available, is_torch_available


__A	:	Tuple     =     {
    'configuration_audio_spectrogram_transformer': [
        'AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'ASTConfig',
    ]
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__A	:	Tuple     =     [
							    'AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'ASTForAudioClassification',
							    'ASTModel',
							    'ASTPreTrainedModel',
							]

try:
							if not is_speech_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__A	:	int     =     ['ASTFeatureExtractor']

if TYPE_CHECKING:
							from .configuration_audio_spectrogram_transformer import (
							    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    ASTConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_audio_spectrogram_transformer import (
														    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ASTForAudioClassification,
														    ASTModel,
														    ASTPreTrainedModel,
														)

							try:
														if not is_speech_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .feature_extraction_audio_spectrogram_transformer import ASTFeatureExtractor


else:
							import sys

							__A	:	Union[str, Any]     =     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)