
'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__ :			List[Any]											=		logging.get_logger(__name__)

UpperCAmelCase__ :			List[str]											=		"▁"

UpperCAmelCase__ :			Dict											=		{"vocab_file": "sentencepiece.bpe.model"}

UpperCAmelCase__ :			int											=		{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

UpperCAmelCase__ :			Dict											=		{
    "facebook/xglm-564M": 20_48,
}






class  A  (						SCREAMING_SNAKE_CASE__     ):
	snake_case__   :List[Any]   =  VOCAB_FILES_NAMES
	snake_case__   :Any   =  PRETRAINED_VOCAB_FILES_MAP
	snake_case__   :str   =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__   :Tuple   =  ['input_ids', 'attention_mask']





	def __init__(       self      :							int   ,				__magic_name__      :							str   ,				__magic_name__      :							Optional[int]="<s>"   ,				__magic_name__      :							int="</s>"   ,				__magic_name__      :							Optional[Any]="</s>"   ,				__magic_name__      :							List[Any]="<s>"   ,				__magic_name__      :							List[str]="<unk>"   ,				__magic_name__      :							Optional[int]="<pad>"   ,				__magic_name__      :							Optional[Dict[str, Any]] = None   ,				**__magic_name__      :							Optional[Any]   ,				):


				"""simple docstring"""

				lowerCAmelCase__										=				{} if sp_model_kwargs is None else sp_model_kwargs

				# Compatibility with the original tokenizer
				lowerCAmelCase__										=				7
				lowerCAmelCase__										=				[f"""<madeupword{i}>""" for i in range(self.num_madeup_words				)]

				lowerCAmelCase__										=				kwargs.get("additional_special_tokens"   ,				[]				)
				kwargs["additional_special_tokens"] += [
				    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    bos_token=__magic_name__   ,				eos_token=__magic_name__   ,				unk_token=__magic_name__   ,				sep_token=__magic_name__   ,				cls_token=__magic_name__   ,				pad_token=__magic_name__   ,				sp_model_kwargs=self.sp_model_kwargs   ,				**__magic_name__   ,				)

				lowerCAmelCase__										=				spm.SentencePieceProcessor(**self.sp_model_kwargs				)
				self.sp_model.Load(str(__magic_name__				)				)
				lowerCAmelCase__										=				vocab_file

				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
				# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
				# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				lowerCAmelCase__										=				1

				# Mimic fairseq token-to-id alignment for the first 4 token
				lowerCAmelCase__										=				{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

				lowerCAmelCase__										=				len(self.sp_model				)
				lowerCAmelCase__										=				{f"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words				)}
				self.fairseq_tokens_to_ids.update(__magic_name__				)

				lowerCAmelCase__										=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}





	def __getstate__(       self      :							List[str]				):


				"""simple docstring"""

				lowerCAmelCase__										=				self.__dict__.copy()
				lowerCAmelCase__										=				None
				lowerCAmelCase__										=				self.sp_model.serialized_model_proto()
				return state





	def __setstate__(       self      :							List[str]   ,				__magic_name__      :							List[str]				):


				"""simple docstring"""

				lowerCAmelCase__										=				d

				# for backward compatibility
				if not hasattr(self   ,				"sp_model_kwargs"				):
							lowerCAmelCase__										=				{}

				lowerCAmelCase__										=				spm.SentencePieceProcessor(**self.sp_model_kwargs				)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Optional[Any]   ,				__magic_name__      :							List[int]   ,				__magic_name__      :							Optional[List[int]] = None				):


				"""simple docstring"""

				if token_ids_a is None:
							return [self.sep_token_id] + token_ids_a
				lowerCAmelCase__										=				[self.sep_token_id]
				return sep + token_ids_a + sep + sep + token_ids_a





	def 				__SCREAMING_SNAKE_CASE	(       self      :							List[str]   ,				__magic_name__      :							List[int]   ,				__magic_name__      :							Optional[List[int]] = None   ,				__magic_name__      :							bool = False				):


				"""simple docstring"""

				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__magic_name__   ,				token_ids_a=__magic_name__   ,				already_has_special_tokens=__magic_name__				)

				if token_ids_a is None:
							return [1] + ([0] * len(__magic_name__				))
				return [1] + ([0] * len(__magic_name__				)) + [1, 1] + ([0] * len(__magic_name__				))





	def 				__SCREAMING_SNAKE_CASE	(       self      :							int   ,				__magic_name__      :							List[int]   ,				__magic_name__      :							Optional[List[int]] = None				):


				"""simple docstring"""

				lowerCAmelCase__										=				[self.sep_token_id]

				if token_ids_a is None:
							return len(sep + token_ids_a				) * [0]
				return len(sep + token_ids_a + sep + sep + token_ids_a				) * [0]





	@property
	def 				__SCREAMING_SNAKE_CASE	(       self      :							List[Any]				):


				"""simple docstring"""

				return len(self.sp_model				) + self.fairseq_offset + self.num_madeup_words





	def 				__SCREAMING_SNAKE_CASE	(       self      :							List[str]				):


				"""simple docstring"""

				lowerCAmelCase__										=				{self.convert_ids_to_tokens(__magic_name__				): i for i in range(self.vocab_size				)}
				vocab.update(self.added_tokens_encoder				)
				return vocab





	def 				__SCREAMING_SNAKE_CASE	(       self      :							str   ,				__magic_name__      :							str				):


				"""simple docstring"""

				return self.sp_model.encode(__magic_name__   ,				out_type=__magic_name__				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Optional[Any]   ,				__magic_name__      :							Dict				):


				"""simple docstring"""

				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				lowerCAmelCase__										=				self.sp_model.PieceToId(__magic_name__				)

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Any   ,				__magic_name__      :							Tuple				):


				"""simple docstring"""

				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Any   ,				__magic_name__      :							List[str]				):


				"""simple docstring"""

				lowerCAmelCase__										=				"".join(__magic_name__				).replace(__magic_name__   ,				" "				).strip()
				return out_string







	def 				__SCREAMING_SNAKE_CASE	(       self      :							Dict   ,				__magic_name__      :							str   ,				__magic_name__      :							Optional[str] = None				):


				"""simple docstring"""

				if not os.path.isdir(__magic_name__				):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
							return
				lowerCAmelCase__										=				os.path.join(
				    __magic_name__   ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

				if os.path.abspath(self.vocab_file				) != os.path.abspath(__magic_name__				) and os.path.isfile(self.vocab_file				):
							copyfile(self.vocab_file   ,				__magic_name__				)
				elif not os.path.isfile(self.vocab_file				):
							with open(__magic_name__   ,				"wb"				) as fi:
										lowerCAmelCase__										=				self.sp_model.serialized_model_proto()
										fi.write(__magic_name__				)

				return (out_vocab_file,)











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__ :   Optional[Any]						=							{
    '''configuration_whisper''': ['''WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''WhisperConfig''', '''WhisperOnnxConfig'''],
    '''feature_extraction_whisper''': ['''WhisperFeatureExtractor'''],
    '''processing_whisper''': ['''WhisperProcessor'''],
    '''tokenization_whisper''': ['''WhisperTokenizer'''],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   str						=							['''WhisperTokenizerFast''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   List[Any]						=							[
						    '''WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''WhisperForConditionalGeneration''',
						    '''WhisperModel''',
						    '''WhisperPreTrainedModel''',
						    '''WhisperForAudioClassification''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   Tuple						=							[
						    '''TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFWhisperForConditionalGeneration''',
						    '''TFWhisperModel''',
						    '''TFWhisperPreTrainedModel''',
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   List[str]						=							[
						    '''FlaxWhisperForConditionalGeneration''',
						    '''FlaxWhisperModel''',
						    '''FlaxWhisperPreTrainedModel''',
						    '''FlaxWhisperForAudioClassification''',
						]


if TYPE_CHECKING:
						from .configuration_whisper import WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP, WhisperConfig, WhisperOnnxConfig
						from .feature_extraction_whisper import WhisperFeatureExtractor
						from .processing_whisper import WhisperProcessor
						from .tokenization_whisper import WhisperTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_whisper_fast import WhisperTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_whisper import (
												    WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    WhisperForAudioClassification,
												    WhisperForConditionalGeneration,
												    WhisperModel,
												    WhisperPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_whisper import (
												    TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFWhisperForConditionalGeneration,
												    TFWhisperModel,
												    TFWhisperPreTrainedModel,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_whisper import (
												    FlaxWhisperForAudioClassification,
												    FlaxWhisperForConditionalGeneration,
												    FlaxWhisperModel,
												    FlaxWhisperPreTrainedModel,
												)

else:
						import sys

						UpperCamelCase__ :   Optional[Any]						=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



