

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_a:    List[str] 							=			logging.get_logger(__name__)

_a:    Dict 							=			{"""vocab_file""": """spiece.model"""}

_a:    Any 							=			{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

_a:    Optional[Any] 							=			{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
_a:    Any 							=			0
_a:    List[str] 							=			1
_a:    Any 							=			2
_a:    str 							=			3
_a:    Dict 							=			4

class 						__UpperCamelCase    (					lowercase						):
	SCREAMING_SNAKE_CASE__    				=     VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE__    				=     PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE__    				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE__    				=     'left'




	def __init__(		self	:						Any		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int=False		,	lowerCAmelCase	:						Union[str, Any]=True		,	lowerCAmelCase	:						Optional[int]=False		,	lowerCAmelCase	:						List[str]="<s>"		,	lowerCAmelCase	:						str="</s>"		,	lowerCAmelCase	:						Tuple="<unk>"		,	lowerCAmelCase	:						List[str]="<sep>"		,	lowerCAmelCase	:						Union[str, Any]="<pad>"		,	lowerCAmelCase	:						Any="<cls>"		,	lowerCAmelCase	:						Any="<mask>"		,	lowerCAmelCase	:						Optional[Any]=["<eop>", "<eod>"]		,	lowerCAmelCase	:						Optional[Dict[str, Any]] = None		,	**lowerCAmelCase	:						Any		,	):


		'''simple docstring'''







		UpperCAmelCase_       				=			AddedToken(lowerCAmelCase		,	lstrip=lowerCAmelCase		,	rstrip=lowerCAmelCase							) if isinstance(lowerCAmelCase		,	lowerCAmelCase							) else mask_token

		UpperCAmelCase_       				=			{} if sp_model_kwargs is None else sp_model_kwargs

		super().__init__(
		    do_lower_case=lowerCAmelCase		,	remove_space=lowerCAmelCase		,	keep_accents=lowerCAmelCase		,	bos_token=lowerCAmelCase		,	eos_token=lowerCAmelCase		,	unk_token=lowerCAmelCase		,	sep_token=lowerCAmelCase		,	pad_token=lowerCAmelCase		,	cls_token=lowerCAmelCase		,	mask_token=lowerCAmelCase		,	additional_special_tokens=lowerCAmelCase		,	sp_model_kwargs=self.sp_model_kwargs		,	**lowerCAmelCase		,	)

		UpperCAmelCase_       				=			3

		UpperCAmelCase_       				=			do_lower_case
		UpperCAmelCase_       				=			remove_space
		UpperCAmelCase_       				=			keep_accents
		UpperCAmelCase_       				=			vocab_file

		UpperCAmelCase_       				=			spm.SentencePieceProcessor(**self.sp_model_kwargs							)
		self.sp_model.Load(lowerCAmelCase							)




	@property
	def       __A					(		self	:						Union[str, Any]							):


		'''simple docstring'''







		return len(self.sp_model							)




	def       __A					(		self	:						Tuple							):


		'''simple docstring'''







		UpperCAmelCase_       				=			{self.convert_ids_to_tokens(lowerCAmelCase							): i for i in range(self.vocab_size							)}
		vocab.update(self.added_tokens_encoder							)
		return vocab




	def __getstate__(		self	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.__dict__.copy()
		UpperCAmelCase_       				=			None
		return state




	def __setstate__(		self	:						str		,	lowerCAmelCase	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			d

		# for backward compatibility
		if not hasattr(self		,	"sp_model_kwargs"							):
			UpperCAmelCase_       				=			{}

		UpperCAmelCase_       				=			spm.SentencePieceProcessor(**self.sp_model_kwargs							)
		self.sp_model.Load(self.vocab_file							)




	def       __A					(		self	:						Optional[int]		,	lowerCAmelCase	:						List[Any]							):


		'''simple docstring'''







		if self.remove_space:
			UpperCAmelCase_       				=			" ".join(inputs.strip().split()							)
		else:
			UpperCAmelCase_       				=			inputs
		UpperCAmelCase_       				=			outputs.replace("``"		,	"\""							).replace("''"		,	"\""							)

		if not self.keep_accents:
			UpperCAmelCase_       				=			unicodedata.normalize("NFKD"		,	lowerCAmelCase							)
			UpperCAmelCase_       				=			"".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase							)]							)
		if self.do_lower_case:
			UpperCAmelCase_       				=			outputs.lower()

		return outputs




	def       __A					(		self	:						Optional[int]		,	lowerCAmelCase	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.preprocess_text(lowerCAmelCase							)
		UpperCAmelCase_       				=			self.sp_model.encode(lowerCAmelCase		,	out_type=lowerCAmelCase							)
		UpperCAmelCase_       				=			[]
		for piece in pieces:
			if len(lowerCAmelCase							) > 1 and piece[-1] == str(","							) and piece[-2].isdigit():
				UpperCAmelCase_       				=			self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase		,	""							)							)
				if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
					if len(cur_pieces[0]							) == 1:
						UpperCAmelCase_       				=			cur_pieces[1:]
					else:
						UpperCAmelCase_       				=			cur_pieces[0][1:]
				cur_pieces.append(piece[-1]							)
				new_pieces.extend(lowerCAmelCase							)
			else:
				new_pieces.append(lowerCAmelCase							)

		return new_pieces




	def       __A					(		self	:						str		,	lowerCAmelCase	:						Tuple							):


		'''simple docstring'''







		return self.sp_model.PieceToId(lowerCAmelCase							)




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						Optional[int]							):


		'''simple docstring'''







		return self.sp_model.IdToPiece(lowerCAmelCase							)




	def       __A					(		self	:						int		,	lowerCAmelCase	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			"".join(lowerCAmelCase							).replace(lowerCAmelCase		,	" "							).strip()
		return out_string




	def       __A					(		self	:						Optional[int]		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						bool = False		,	lowerCAmelCase	:						bool = None		,	lowerCAmelCase	:						bool = True		,	**lowerCAmelCase	:						Dict		,	):


		'''simple docstring'''







		UpperCAmelCase_       				=			kwargs.pop("use_source_tokenizer"		,	lowerCAmelCase							)

		UpperCAmelCase_       				=			self.convert_ids_to_tokens(lowerCAmelCase		,	skip_special_tokens=lowerCAmelCase							)

		# To avoid mixing byte-level and unicode for byte-level BPT
		# we need to build string separately for added tokens and byte-level tokens
		# cf. https://github.com/huggingface/transformers/issues/1133
		UpperCAmelCase_       				=			[]
		UpperCAmelCase_       				=			[]
		for token in filtered_tokens:
			if skip_special_tokens and token in self.all_special_ids:
				continue
			if token in self.added_tokens_encoder:
				if current_sub_text:
					sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase							)							)
					UpperCAmelCase_       				=			[]
				sub_texts.append(lowerCAmelCase							)
			else:
				current_sub_text.append(lowerCAmelCase							)
		if current_sub_text:
			sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase							)							)

		# Mimic the behavior of the Rust tokenizer:
		# By default, there are no spaces between special tokens
		UpperCAmelCase_       				=			"".join(lowerCAmelCase							)

		UpperCAmelCase_       				=			(
		    clean_up_tokenization_spaces
		    if clean_up_tokenization_spaces is not None
		    else self.clean_up_tokenization_spaces
		)
		if clean_up_tokenization_spaces:
			UpperCAmelCase_       				=			self.clean_up_tokenization(lowerCAmelCase							)
			return clean_text
		else:
			return text




	def       __A					(		self	:						str		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None							):


		'''simple docstring'''







		UpperCAmelCase_       				=			[self.sep_token_id]
		UpperCAmelCase_       				=			[self.cls_token_id]
		if token_ids_a is None:
			return token_ids_a + sep + cls
		return token_ids_a + sep + token_ids_a + sep + cls




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None		,	lowerCAmelCase	:						bool = False							):


		'''simple docstring'''







		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=lowerCAmelCase		,	token_ids_a=lowerCAmelCase		,	already_has_special_tokens=lowerCAmelCase							)

		if token_ids_a is not None:
			return ([0] * len(lowerCAmelCase							)) + [1] + ([0] * len(lowerCAmelCase							)) + [1, 1]
		return ([0] * len(lowerCAmelCase							)) + [1, 1]




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None							):


		'''simple docstring'''







		UpperCAmelCase_       				=			[self.sep_token_id]
		UpperCAmelCase_       				=			[2]

		if token_ids_a is None:
			return len(token_ids_a + sep							) * [0] + cls_segment_id
		return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Optional[str] = None							):


		'''simple docstring'''







		if not os.path.isdir(lowerCAmelCase							):
			logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
			return
		UpperCAmelCase_       				=			os.path.join(
		    lowerCAmelCase		,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

		if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCAmelCase							) and os.path.isfile(self.vocab_file							):
			copyfile(self.vocab_file		,	lowerCAmelCase							)
		elif not os.path.isfile(self.vocab_file							):
			with open(lowerCAmelCase		,	"wb"							) as fi:
				UpperCAmelCase_       				=			self.sp_model.serialized_model_proto()
				fi.write(lowerCAmelCase							)

		return (out_vocab_file,)


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


_a:    List[str] 							=			{
    """configuration_encodec""": [
        """ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """EncodecConfig""",
    ],
    """feature_extraction_encodec""": ["""EncodecFeatureExtractor"""],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_a:    List[Any] 							=			[
				    """ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST""",
				    """EncodecModel""",
				    """EncodecPreTrainedModel""",
				]

if TYPE_CHECKING:
				from .configuration_encodec import (
				    ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP,
				    EncodecConfig,
				)
				from .feature_extraction_encodec import EncodecFeatureExtractor

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_encodec import (
								    ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST,
								    EncodecModel,
								    EncodecPreTrainedModel,
								)

else:
				import sys

				_a:    int 							=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)