

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_snake_case						: Optional[int]											=							logging.get_logger(__name__)

_snake_case						: int											=							{'vocab_file': 'spiece.model'}

_snake_case						: Optional[int]											=							{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

_snake_case						: Tuple											=							{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
_snake_case						: int											=							0
_snake_case						: str											=							1
_snake_case						: Any											=							2
_snake_case						: Any											=							3
_snake_case						: int											=							4





class __SCREAMING_SNAKE_CASE     (	__UpperCAmelCase		):
    SCREAMING_SNAKE_CASE__      =VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE__      =PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE__      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE__      ='''left'''






    def __init__(       self,		_a,		_a=False,		_a=True,		_a=False,		_a="<s>",		_a="</s>",		_a="<unk>",		_a="<sep>",		_a="<pad>",		_a="<cls>",		_a="<mask>",		_a=["<eop>", "<eod>"],		_a = None,		**_a,		)   ->					None:
       # Mask token behave like a normal word, i.e. include the space before it
       __SCREAMING_SNAKE_CASE       =				AddedToken(lowerCamelCase__,		lstrip=lowerCamelCase__,		rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__,		lowerCamelCase__ ) else mask_token

       __SCREAMING_SNAKE_CASE       =				{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=lowerCamelCase__,		remove_space=lowerCamelCase__,		keep_accents=lowerCamelCase__,		bos_token=lowerCamelCase__,		eos_token=lowerCamelCase__,		unk_token=lowerCamelCase__,		sep_token=lowerCamelCase__,		pad_token=lowerCamelCase__,		cls_token=lowerCamelCase__,		mask_token=lowerCamelCase__,		additional_special_tokens=lowerCamelCase__,		sp_model_kwargs=self.sp_model_kwargs,		**lowerCamelCase__,		)

       __SCREAMING_SNAKE_CASE       =				3

       __SCREAMING_SNAKE_CASE       =				do_lower_case
       __SCREAMING_SNAKE_CASE       =				remove_space
       __SCREAMING_SNAKE_CASE       =				keep_accents
       __SCREAMING_SNAKE_CASE       =				vocab_file

       __SCREAMING_SNAKE_CASE       =				spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(lowerCamelCase__ )






    @property
    def 						__lowerCAmelCase    (       self )   ->					List[Any]:
       return len(self.sp_model )






    def 						__lowerCAmelCase    (       self )   ->					int:
       __SCREAMING_SNAKE_CASE       =				{self.convert_ids_to_tokens(lowerCamelCase__ ): i for i in range(self.vocab_size )}
       vocab.update(self.added_tokens_encoder )
       return vocab






    def __getstate__(       self )   ->					Tuple:
       __SCREAMING_SNAKE_CASE       =				self.__dict__.copy()
       __SCREAMING_SNAKE_CASE       =				None
       return state






    def __setstate__(       self,		_a )   ->					Optional[Any]:
       __SCREAMING_SNAKE_CASE       =				d

       # for backward compatibility
       if not hasattr(self,		"sp_model_kwargs" ):
          __SCREAMING_SNAKE_CASE       =				{}

       __SCREAMING_SNAKE_CASE       =				spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(self.vocab_file )






    def 						__lowerCAmelCase    (       self,		_a )   ->					str:
       if self.remove_space:
          __SCREAMING_SNAKE_CASE       =				""" """.join(inputs.strip().split() )
       else:
          __SCREAMING_SNAKE_CASE       =				inputs
       __SCREAMING_SNAKE_CASE       =				outputs.replace("``",		"\"" ).replace("''",		"\"" )

       if not self.keep_accents:
          __SCREAMING_SNAKE_CASE       =				unicodedata.normalize("NFKD",		lowerCamelCase__ )
          __SCREAMING_SNAKE_CASE       =				"""""".join([c for c in outputs if not unicodedata.combining(lowerCamelCase__ )] )
       if self.do_lower_case:
          __SCREAMING_SNAKE_CASE       =				outputs.lower()

       return outputs






    def 						__lowerCAmelCase    (       self,		_a )   ->					List[str]:
       __SCREAMING_SNAKE_CASE       =				self.preprocess_text(lowerCamelCase__ )
       __SCREAMING_SNAKE_CASE       =				self.sp_model.encode(lowerCamelCase__,		out_type=lowerCamelCase__ )
       __SCREAMING_SNAKE_CASE       =				[]
       for piece in pieces:
          if len(lowerCamelCase__ ) > 1 and piece[-1] == str("," ) and piece[-2].isdigit():
             __SCREAMING_SNAKE_CASE       =				self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase__,		"" ) )
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                if len(cur_pieces[0] ) == 1:
                   __SCREAMING_SNAKE_CASE       =				cur_pieces[1:]
                else:
                   __SCREAMING_SNAKE_CASE       =				cur_pieces[0][1:]
             cur_pieces.append(piece[-1] )
             new_pieces.extend(lowerCamelCase__ )
          else:
             new_pieces.append(lowerCamelCase__ )

       return new_pieces






    def 						__lowerCAmelCase    (       self,		_a )   ->					Union[str, Any]:
       return self.sp_model.PieceToId(lowerCamelCase__ )






    def 						__lowerCAmelCase    (       self,		_a )   ->					Union[str, Any]:
       return self.sp_model.IdToPiece(lowerCamelCase__ )






    def 						__lowerCAmelCase    (       self,		_a )   ->					List[Any]:
       __SCREAMING_SNAKE_CASE       =				"""""".join(lowerCamelCase__ ).replace(lowerCamelCase__,		" " ).strip()
       return out_string






    def 						__lowerCAmelCase    (       self,		_a,		_a = False,		_a = None,		_a = True,		**_a,		)   ->					str:
       __SCREAMING_SNAKE_CASE       =				kwargs.pop("use_source_tokenizer",		lowerCamelCase__ )

       __SCREAMING_SNAKE_CASE       =				self.convert_ids_to_tokens(lowerCamelCase__,		skip_special_tokens=lowerCamelCase__ )

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       __SCREAMING_SNAKE_CASE       =				[]
       __SCREAMING_SNAKE_CASE       =				[]
       for token in filtered_tokens:
          if skip_special_tokens and token in self.all_special_ids:
             continue
          if token in self.added_tokens_encoder:
             if current_sub_text:
                sub_texts.append(self.convert_tokens_to_string(lowerCamelCase__ ) )
                __SCREAMING_SNAKE_CASE       =				[]
             sub_texts.append(lowerCamelCase__ )
          else:
             current_sub_text.append(lowerCamelCase__ )
       if current_sub_text:
          sub_texts.append(self.convert_tokens_to_string(lowerCamelCase__ ) )

       # Mimic the behavior of the Rust tokenizer:
       # By default, there are no spaces between special tokens
       __SCREAMING_SNAKE_CASE       =				"""""".join(lowerCamelCase__ )

       __SCREAMING_SNAKE_CASE       =				(
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
          __SCREAMING_SNAKE_CASE       =				self.clean_up_tokenization(lowerCamelCase__ )
          return clean_text
       else:
          return text






    def 						__lowerCAmelCase    (       self,		_a,		_a = None )   ->					List[int]:
       __SCREAMING_SNAKE_CASE       =				[self.sep_token_id]
       __SCREAMING_SNAKE_CASE       =				[self.cls_token_id]
       if token_ids_a is None:
          return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls






    def 						__lowerCAmelCase    (       self,		_a,		_a = None,		_a = False )   ->					List[int]:

       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=lowerCamelCase__,		token_ids_a=lowerCamelCase__,		already_has_special_tokens=lowerCamelCase__ )

       if token_ids_a is not None:
          return ([0] * len(lowerCamelCase__ )) + [1] + ([0] * len(lowerCamelCase__ )) + [1, 1]
       return ([0] * len(lowerCamelCase__ )) + [1, 1]






    def 						__lowerCAmelCase    (       self,		_a,		_a = None )   ->					List[int]:
       __SCREAMING_SNAKE_CASE       =				[self.sep_token_id]
       __SCREAMING_SNAKE_CASE       =				[2]

       if token_ids_a is None:
          return len(token_ids_a + sep ) * [0] + cls_segment_id
       return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id






    def 						__lowerCAmelCase    (       self,		_a,		_a = None )   ->					Tuple[str]:
       if not os.path.isdir(lowerCamelCase__ ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
          return
       __SCREAMING_SNAKE_CASE       =				os.path.join(
           lowerCamelCase__,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

       if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCamelCase__ ) and os.path.isfile(self.vocab_file ):
          copyfile(self.vocab_file,		lowerCamelCase__ )
       elif not os.path.isfile(self.vocab_file ):
          with open(lowerCamelCase__,		"wb" ) as fi:
             __SCREAMING_SNAKE_CASE       =				self.sp_model.serialized_model_proto()
             fi.write(lowerCamelCase__ )

       return (out_vocab_file,)





"""simple docstring"""


from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__snake_case    					=					{
    'configuration_informer': [
        'INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'InformerConfig',
    ],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __snake_case    					=					[
       'INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
       'InformerForPrediction',
       'InformerModel',
       'InformerPreTrainedModel',
   ]


if TYPE_CHECKING:
   from .configuration_informer import INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, InformerConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_informer import (
          INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          InformerForPrediction,
          InformerModel,
          InformerPreTrainedModel,
      )

else:
   import sys

   __snake_case    					=					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)