






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase			:		str   				=    logging.get_logger(__name__)


_UpperCAmelCase			:		Optional[int]   				=    "▁"

_UpperCAmelCase			:		Dict   				=    {"vocab_file": "spiece.model"}

_UpperCAmelCase			:		Any   				=    {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

_UpperCAmelCase			:		Tuple   				=    {
    "google/reformer-crime-and-punishment": 524_288,
}
class  lowerCAmelCase_       (				lowerCAmelCase__					):
							UpperCamelCase_		:Tuple		      =    VOCAB_FILES_NAMES
							UpperCamelCase_		:List[Any]		      =    PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_		:Union[str, Any]		      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_		:Dict		      =    ["input_ids", "attention_mask"]




							def __init__(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]="</s>" ,							SCREAMING_SNAKE_CASE_     :   Tuple="<unk>" ,							SCREAMING_SNAKE_CASE_     :   int=[] ,							SCREAMING_SNAKE_CASE_     :   List[Any] = None ,							**SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							):
											lowerCAmelCase__      			=						{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    eos_token=_lowerCamelCase ,							unk_token=_lowerCamelCase ,							additional_special_tokens=_lowerCamelCase ,							sp_model_kwargs=self.sp_model_kwargs ,							**_lowerCamelCase ,							)

											lowerCAmelCase__      			=						vocab_file
											lowerCAmelCase__      			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(_lowerCamelCase	)




							@property
							def 						__snake_case					(   self     :   int	):
											return self.sp_model.get_piece_size()




							def 						__snake_case					(   self     :   Any	):
											lowerCAmelCase__      			=						{self.convert_ids_to_tokens(_lowerCamelCase	): i for i in range(self.vocab_size	)}
											vocab.update(self.added_tokens_encoder	)
											return vocab




							def __getstate__(   self     :   int	):
											lowerCAmelCase__      			=						self.__dict__.copy()
											lowerCAmelCase__      			=						None
											return state




							def __setstate__(   self     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Any	):
											lowerCAmelCase__      			=						d

											# for backward compatibility
											if not hasattr(self ,							'''sp_model_kwargs'''	):
															lowerCAmelCase__      			=						{}

											lowerCAmelCase__      			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(self.vocab_file	)




							def 						__snake_case					(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]	):
											return self.sp_model.encode(_lowerCamelCase ,							out_type=_lowerCamelCase	)




							def 						__snake_case					(   self     :   Any ,							SCREAMING_SNAKE_CASE_     :   Any	):
											return self.sp_model.piece_to_id(_lowerCamelCase	)




							def 						__snake_case					(   self     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[int]	):





											if index < self.sp_model.get_piece_size():
															lowerCAmelCase__      			=						self.sp_model.IdToPiece(_lowerCamelCase	)
											return token




							def 						__snake_case					(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   int	):
											lowerCAmelCase__      			=						[]
											lowerCAmelCase__      			=						''''''
											for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																			out_string += self.sp_model.decode(_lowerCamelCase	) + token
																			lowerCAmelCase__      			=						[]
															else:
																			current_sub_tokens.append(_lowerCamelCase	)
											out_string += self.sp_model.decode(_lowerCamelCase	)
											return out_string.strip()




							def 						__snake_case					(   self     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] = None	):





											if not os.path.isdir(_lowerCamelCase	):
															logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
															return
											lowerCAmelCase__      			=						os.path.join(
											    _lowerCamelCase ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

											if os.path.abspath(self.vocab_file	) != os.path.abspath(_lowerCamelCase	) and os.path.isfile(self.vocab_file	):
															copyfile(self.vocab_file ,							_lowerCamelCase	)
											elif not os.path.isfile(self.vocab_file	):
															with open(_lowerCamelCase ,							'''wb'''	) as fi:
																			lowerCAmelCase__      			=						self.sp_model.serialized_model_proto()
																			fi.write(_lowerCamelCase	)

											return (out_vocab_file,)













from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCAmelCase			:		Optional[Any]   				=    {
    "configuration_mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig"],
    "feature_extraction_mctct": ["MCTCTFeatureExtractor"],
    "processing_mctct": ["MCTCTProcessor"],
}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		str   				=    [
			    "MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "MCTCTForCTC",
			    "MCTCTModel",
			    "MCTCTPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
			from .feature_extraction_mctct import MCTCTFeatureExtractor
			from .processing_mctct import MCTCTProcessor

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
			import sys

			_UpperCAmelCase			:		List[str]   				=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





