
'''simple docstring'''
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__lowerCAmelCase												=	logging.get_logger(__name__)

__lowerCAmelCase												=	{'vocab_file': 'spiece.model'}

__lowerCAmelCase												=	{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

__lowerCAmelCase												=	{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
__lowerCAmelCase												=	0
__lowerCAmelCase												=	1
__lowerCAmelCase												=	2
__lowerCAmelCase												=	3
__lowerCAmelCase												=	4


class       _lowerCAmelCase   (   __snake_case				):







							'''simple docstring'''

							lowerCAmelCase_ 						=						VOCAB_FILES_NAMES
							lowerCAmelCase_ 						=						PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase_ 						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase_ 						=						"left"

							def __init__(self  ,  UpperCAmelCase  ,  UpperCAmelCase=False  ,  UpperCAmelCase=True  ,  UpperCAmelCase=False  ,  UpperCAmelCase="<s>"  ,  UpperCAmelCase="</s>"  ,  UpperCAmelCase="<unk>"  ,  UpperCAmelCase="<sep>"  ,  UpperCAmelCase="<pad>"  ,  UpperCAmelCase="<cls>"  ,  UpperCAmelCase="<mask>"  ,  UpperCAmelCase=["<eop>", "<eod>"]  ,  UpperCAmelCase = None  ,  **UpperCAmelCase  ,  )					->       None:
														# Mask token behave like a normal word, i.e. include the space before it
														_snake_case       =  AddedToken(UpperCAmelCase  ,  lstrip=UpperCAmelCase  ,  rstrip=UpperCAmelCase  ) if isinstance(UpperCAmelCase  ,  UpperCAmelCase  ) else mask_token

														_snake_case       =  {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=UpperCAmelCase  ,  remove_space=UpperCAmelCase  ,  keep_accents=UpperCAmelCase  ,  bos_token=UpperCAmelCase  ,  eos_token=UpperCAmelCase  ,  unk_token=UpperCAmelCase  ,  sep_token=UpperCAmelCase  ,  pad_token=UpperCAmelCase  ,  cls_token=UpperCAmelCase  ,  mask_token=UpperCAmelCase  ,  additional_special_tokens=UpperCAmelCase  ,  sp_model_kwargs=self.sp_model_kwargs  ,  **UpperCAmelCase  ,  )

														_snake_case       =  3

														_snake_case       =  do_lower_case
														_snake_case       =  remove_space
														_snake_case       =  keep_accents
														_snake_case       =  vocab_file

														_snake_case       =  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(UpperCAmelCase  )

							@property
							def 	lowercase							(self  )					->       Optional[int]:
														return len(self.sp_model  )

							def 	lowercase							(self  )					->       Optional[int]:
														_snake_case       =  {self.convert_ids_to_tokens(UpperCAmelCase  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab

							def __getstate__(self  )					->       Union[str, Any]:
														_snake_case       =  self.__dict__.copy()
														_snake_case       =  None
														return state

							def __setstate__(self  ,  UpperCAmelCase  )					->       Optional[Any]:
														_snake_case       =  d

														# for backward compatibility
														if not hasattr(self  ,  """sp_model_kwargs"""  ):
																					_snake_case       =  {}

														_snake_case       =  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(self.vocab_file  )

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Dict:
														if self.remove_space:
																					_snake_case       =  """ """.join(inputs.strip().split()  )
														else:
																					_snake_case       =  inputs
														_snake_case       =  outputs.replace("""``"""  ,  """\""""  ).replace("""''"""  ,  """\""""  )

														if not self.keep_accents:
																					_snake_case       =  unicodedata.normalize("""NFKD"""  ,  UpperCAmelCase  )
																					_snake_case       =  """""".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase  )]  )
														if self.do_lower_case:
																					_snake_case       =  outputs.lower()

														return outputs

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       List[str]:
														_snake_case       =  self.preprocess_text(UpperCAmelCase  )
														_snake_case       =  self.sp_model.encode(UpperCAmelCase  ,  out_type=UpperCAmelCase  )
														_snake_case       =  []
														for piece in pieces:
																					if len(UpperCAmelCase  ) > 1 and piece[-1] == str(""","""  ) and piece[-2].isdigit():
																												_snake_case       =  self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase  ,  """"""  )  )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]  ) == 1:
																																										_snake_case       =  cur_pieces[1:]
																																			else:
																																										_snake_case       =  cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]  )
																												new_pieces.extend(UpperCAmelCase  )
																					else:
																												new_pieces.append(UpperCAmelCase  )

														return new_pieces

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Optional[Any]:
														return self.sp_model.PieceToId(UpperCAmelCase  )

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Optional[int]:
														return self.sp_model.IdToPiece(UpperCAmelCase  )

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Tuple:
														_snake_case       =  """""".join(UpperCAmelCase  ).replace(UpperCAmelCase  ,  """ """  ).strip()
														return out_string

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = False  ,  UpperCAmelCase = None  ,  UpperCAmelCase = True  ,  **UpperCAmelCase  ,  )					->       str:
														_snake_case       =  kwargs.pop("""use_source_tokenizer"""  ,  UpperCAmelCase  )

														_snake_case       =  self.convert_ids_to_tokens(UpperCAmelCase  ,  skip_special_tokens=UpperCAmelCase  )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														_snake_case       =  []
														_snake_case       =  []
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase  )  )
																																			_snake_case       =  []
																												sub_texts.append(UpperCAmelCase  )
																					else:
																												current_sub_text.append(UpperCAmelCase  )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase  )  )

														# Mimic the behavior of the Rust tokenizer:
														# By default, there are no spaces between special tokens
														_snake_case       =  """""".join(UpperCAmelCase  )

														_snake_case       =  (
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					_snake_case       =  self.clean_up_tokenization(UpperCAmelCase  )
																					return clean_text
														else:
																					return text

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  )					->       List[int]:
														_snake_case       =  [self.sep_token_id]
														_snake_case       =  [self.cls_token_id]
														if token_ids_a is None:
																					return token_ids_a + sep + cls
														return token_ids_a + sep + token_ids_a + sep + cls

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  ,  UpperCAmelCase = False  )					->       List[int]:

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCAmelCase  ,  token_ids_a=UpperCAmelCase  ,  already_has_special_tokens=UpperCAmelCase  )

														if token_ids_a is not None:
																					return ([0] * len(UpperCAmelCase  )) + [1] + ([0] * len(UpperCAmelCase  )) + [1, 1]
														return ([0] * len(UpperCAmelCase  )) + [1, 1]

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  )					->       List[int]:
														_snake_case       =  [self.sep_token_id]
														_snake_case       =  [2]

														if token_ids_a is None:
																					return len(token_ids_a + sep  ) * [0] + cls_segment_id
														return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id





							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  )					->       Tuple[str]:
														if not os.path.isdir(UpperCAmelCase  ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														_snake_case       =  os.path.join(
														    UpperCAmelCase  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCAmelCase  ) and os.path.isfile(self.vocab_file  ):
																					copyfile(self.vocab_file  ,  UpperCAmelCase  )
														elif not os.path.isfile(self.vocab_file  ):
																					with open(UpperCAmelCase  ,  """wb"""  ) as fi:
																												_snake_case       =  self.sp_model.serialized_model_proto()
																												fi.write(UpperCAmelCase  )

														return (out_vocab_file,)

'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


__lowerCAmelCase												=	{
    'configuration_efficientformer': [
        'EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'EfficientFormerConfig',
    ]
}

try:
				if not is_vision_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__lowerCAmelCase												=	['EfficientFormerImageProcessor']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__lowerCAmelCase												=	[
				    'EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'EfficientFormerForImageClassification',
				    'EfficientFormerForImageClassificationWithTeacher',
				    'EfficientFormerModel',
				    'EfficientFormerPreTrainedModel',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__lowerCAmelCase												=	[
				    'TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'TFEfficientFormerForImageClassification',
				    'TFEfficientFormerForImageClassificationWithTeacher',
				    'TFEfficientFormerModel',
				    'TFEfficientFormerPreTrainedModel',
				]

if TYPE_CHECKING:
				from .configuration_efficientformer import EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, EfficientFormerConfig

				try:
								if not is_vision_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .image_processing_efficientformer import EfficientFormerImageProcessor

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_efficientformer import (
								    EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    EfficientFormerForImageClassification,
								    EfficientFormerForImageClassificationWithTeacher,
								    EfficientFormerModel,
								    EfficientFormerPreTrainedModel,
								)
				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_efficientformer import (
								    TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFEfficientFormerForImageClassification,
								    TFEfficientFormerForImageClassificationWithTeacher,
								    TFEfficientFormerModel,
								    TFEfficientFormerPreTrainedModel,
								)

else:
				import sys

				__lowerCAmelCase												=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)