





"""simple docstring"""
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__UpperCamelCase     :						int						       =  logging.get_logger(__name__)

__UpperCamelCase     :						Optional[Any]						       =  {'''vocab_file''': '''spiece.model'''}

__UpperCamelCase     :						int						       =  {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

__UpperCamelCase     :						str						       =  {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
__UpperCamelCase     :						Any						       =  0
__UpperCamelCase     :						str						       =  1
__UpperCamelCase     :						List[Any]						       =  2
__UpperCamelCase     :						Dict						       =  3
__UpperCamelCase     :						Dict						       =  4







class       a							(   a__  ):
 snake_case__             =       VOCAB_FILES_NAMES
 snake_case__             =       PRETRAINED_VOCAB_FILES_MAP
 snake_case__             =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 snake_case__             =       '''left'''



 def __init__(     self				,       _snake_case				,       _snake_case=False				,       _snake_case=True				,       _snake_case=False				,       _snake_case="<s>"				,       _snake_case="</s>"				,       _snake_case="<unk>"				,       _snake_case="<sep>"				,       _snake_case="<pad>"				,       _snake_case="<cls>"				,       _snake_case="<mask>"				,       _snake_case=["<eop>", "<eod>"]				,       _snake_case = None				,       **_snake_case				,       ):







  """simple docstring"""






  lowerCAmelCase    				= AddedToken(_snake_case				,       lstrip=_snake_case				,       rstrip=_snake_case     ) if isinstance(_snake_case				,       _snake_case     ) else mask_token

  lowerCAmelCase    				= {} if sp_model_kwargs is None else sp_model_kwargs

  super().__init__(
      do_lower_case=_snake_case				,       remove_space=_snake_case				,       keep_accents=_snake_case				,       bos_token=_snake_case				,       eos_token=_snake_case				,       unk_token=_snake_case				,       sep_token=_snake_case				,       pad_token=_snake_case				,       cls_token=_snake_case				,       mask_token=_snake_case				,       additional_special_tokens=_snake_case				,       sp_model_kwargs=self.sp_model_kwargs				,       **_snake_case				,       )

  lowerCAmelCase    				= 3

  lowerCAmelCase    				= do_lower_case
  lowerCAmelCase    				= remove_space
  lowerCAmelCase    				= keep_accents
  lowerCAmelCase    				= vocab_file

  lowerCAmelCase    				= spm.SentencePieceProcessor(**self.sp_model_kwargs     )
  self.sp_model.Load(_snake_case     )



 @property
 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return len(self.sp_model     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= {self.convert_ids_to_tokens(_snake_case     ): i for i in range(self.vocab_size     )}
  vocab.update(self.added_tokens_encoder     )
  return vocab



 def __getstate__(     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.__dict__.copy()
  lowerCAmelCase    				= None
  return state



 def __setstate__(     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= d

  # for backward compatibility
  if not hasattr(self				,       'sp_model_kwargs'     ):
   lowerCAmelCase    				= {}

  lowerCAmelCase    				= spm.SentencePieceProcessor(**self.sp_model_kwargs     )
  self.sp_model.Load(self.vocab_file     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  if self.remove_space:
   lowerCAmelCase    				= ' '.join(inputs.strip().split()     )
  else:
   lowerCAmelCase    				= inputs
  lowerCAmelCase    				= outputs.replace('``'				,       '"'     ).replace('\'\''				,       '"'     )

  if not self.keep_accents:
   lowerCAmelCase    				= unicodedata.normalize('NFKD'				,       _snake_case     )
   lowerCAmelCase    				= ''.join([c for c in outputs if not unicodedata.combining(_snake_case     )]     )
  if self.do_lower_case:
   lowerCAmelCase    				= outputs.lower()

  return outputs



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= self.preprocess_text(_snake_case     )
  lowerCAmelCase    				= self.sp_model.encode(_snake_case				,       out_type=_snake_case     )
  lowerCAmelCase    				= []
  for piece in pieces:
   if len(_snake_case     ) > 1 and piece[-1] == str(','     ) and piece[-2].isdigit():
    lowerCAmelCase    				= self.sp_model.EncodeAsPieces(piece[:-1].replace(_snake_case				,       ''     )     )
    if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
     if len(cur_pieces[0]     ) == 1:
      lowerCAmelCase    				= cur_pieces[1:]
     else:
      lowerCAmelCase    				= cur_pieces[0][1:]
    cur_pieces.append(piece[-1]     )
    new_pieces.extend(_snake_case     )
   else:
    new_pieces.append(_snake_case     )

  return new_pieces



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  return self.sp_model.PieceToId(_snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  return self.sp_model.IdToPiece(_snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= ''.join(_snake_case     ).replace(_snake_case				,       ' '     ).strip()
  return out_string



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = False				,       _snake_case = None				,       _snake_case = True				,       **_snake_case				,       ):







  """simple docstring"""






  lowerCAmelCase    				= kwargs.pop('use_source_tokenizer'				,       _snake_case     )

  lowerCAmelCase    				= self.convert_ids_to_tokens(_snake_case				,       skip_special_tokens=_snake_case     )

  # To avoid mixing byte-level and unicode for byte-level BPT
  # we need to build string separately for added tokens and byte-level tokens
  # cf. https://github.com/huggingface/transformers/issues/1133
  lowerCAmelCase    				= []
  lowerCAmelCase    				= []
  for token in filtered_tokens:
   if skip_special_tokens and token in self.all_special_ids:
    continue
   if token in self.added_tokens_encoder:
    if current_sub_text:
     sub_texts.append(self.convert_tokens_to_string(_snake_case     )     )
     lowerCAmelCase    				= []
    sub_texts.append(_snake_case     )
   else:
    current_sub_text.append(_snake_case     )
  if current_sub_text:
   sub_texts.append(self.convert_tokens_to_string(_snake_case     )     )

  # Mimic the behavior of the Rust tokenizer:
  # By default, there are no spaces between special tokens
  lowerCAmelCase    				= ''.join(_snake_case     )

  lowerCAmelCase    				= (
      clean_up_tokenization_spaces
      if clean_up_tokenization_spaces is not None
      else self.clean_up_tokenization_spaces
  )
  if clean_up_tokenization_spaces:
   lowerCAmelCase    				= self.clean_up_tokenization(_snake_case     )
   return clean_text
  else:
   return text



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = None     ):







  """simple docstring"""






  lowerCAmelCase    				= [self.sep_token_id]
  lowerCAmelCase    				= [self.cls_token_id]
  if token_ids_a is None:
   return token_ids_a + sep + cls
  return token_ids_a + sep + token_ids_a + sep + cls



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = None				,       _snake_case = False     ):







  """simple docstring"""






  if already_has_special_tokens:
   return super().get_special_tokens_mask(
       token_ids_a=_snake_case				,       token_ids_a=_snake_case				,       already_has_special_tokens=_snake_case     )

  if token_ids_a is not None:
   return ([0] * len(_snake_case     )) + [1] + ([0] * len(_snake_case     )) + [1, 1]
  return ([0] * len(_snake_case     )) + [1, 1]



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = None     ):







  """simple docstring"""






  lowerCAmelCase    				= [self.sep_token_id]
  lowerCAmelCase    				= [2]

  if token_ids_a is None:
   return len(token_ids_a + sep     ) * [0] + cls_segment_id
  return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case = None     ):







  """simple docstring"""






  if not os.path.isdir(_snake_case     ):
   logger.error(F'Vocabulary path ({save_directory}) should be a directory'     )
   return
  lowerCAmelCase    				= os.path.join(
      _snake_case				,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

  if os.path.abspath(self.vocab_file     ) != os.path.abspath(_snake_case     ) and os.path.isfile(self.vocab_file     ):
   copyfile(self.vocab_file				,       _snake_case     )
  elif not os.path.isfile(self.vocab_file     ):
   with open(_snake_case				,       'wb'     ) as fi:
    lowerCAmelCase    				= self.sp_model.serialized_model_proto()
    fi.write(_snake_case     )

  return (out_vocab_file,)










"""simple docstring"""







from ..utils import DummyObject, requires_backends

class       _a  ( metaclass=SCREAMING_SNAKE_CASE__):
			__magic_name__     					=     ["""keras_nlp"""]



			def __init__(			self							:				int   ,	*_lowercase							:				Optional[int]   ,	**_lowercase							:				Dict		)							->					List[Any]:
									requires_backends(self   ,	["keras_nlp"]		)



