

"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__UpperCAmelCase			=					logging.get_logger(__name__)

__UpperCAmelCase			=					{'vocab_file': 'spiece.model'}

__UpperCAmelCase			=					{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

__UpperCAmelCase			=					{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
__UpperCAmelCase			=					0
__UpperCAmelCase			=					1
__UpperCAmelCase			=					2
__UpperCAmelCase			=					3
__UpperCAmelCase			=					4







class   __lowercase      (     __lowerCamelCase       ):
  snake_case_												=							VOCAB_FILES_NAMES
  snake_case_												=							PRETRAINED_VOCAB_FILES_MAP
  snake_case_												=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_												=							"""left"""

  def __init__(	self		:      List[Any]       ,A		:      Optional[int]       ,A		:      List[Any]=False       ,A		:      List[Any]=True       ,A		:      Optional[int]=False       ,A		:      Optional[int]="<s>"       ,A		:      List[str]="</s>"       ,A		:      List[str]="<unk>"       ,A		:      Dict="<sep>"       ,A		:      Dict="<pad>"       ,A		:      Tuple="<cls>"       ,A		:      str="<mask>"       ,A		:      Any=["<eop>", "<eod>"]       ,A		:      Optional[Dict[str, Any]] = None       ,**A		:      Tuple       ,):






   '''simple docstring'''
   # Mask token behave like a normal word, i.e. include the space before it
   UpperCAmelCase__       :	Any        =       AddedToken(A       ,lstrip=A       ,rstrip=A     ) if isinstance(A       ,A     ) else mask_token

   UpperCAmelCase__       :	List[Any]        =       {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=A       ,remove_space=A       ,keep_accents=A       ,bos_token=A       ,eos_token=A       ,unk_token=A       ,sep_token=A       ,pad_token=A       ,cls_token=A       ,mask_token=A       ,additional_special_tokens=A       ,sp_model_kwargs=self.sp_model_kwargs       ,**A       ,)

   UpperCAmelCase__       :	Dict        =       3

   UpperCAmelCase__       :	List[str]        =       do_lower_case
   UpperCAmelCase__       :	List[Any]        =       remove_space
   UpperCAmelCase__       :	List[Any]        =       keep_accents
   UpperCAmelCase__       :	Tuple        =       vocab_file

   UpperCAmelCase__       :	str        =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(A     )

  @property
  def  __lowercase							(	self		:      Optional[int]     ):






   '''simple docstring'''
   return len(self.sp_model     )

  def  __lowercase							(	self		:      Optional[int]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       {self.convert_ids_to_tokens(A     ): i for i in range(self.vocab_size     )}
   vocab.update(self.added_tokens_encoder     )
   return vocab

  def __getstate__(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       self.__dict__.copy()
   UpperCAmelCase__       :	Optional[Any]        =       None
   return state

  def __setstate__(	self		:      Tuple       ,A		:      Optional[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Any        =       d

   # for backward compatibility
   if not hasattr(self       ,"""sp_model_kwargs"""     ):
    UpperCAmelCase__       :	Dict        =       {}

   UpperCAmelCase__       :	Any        =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.vocab_file     )

  def  __lowercase							(	self		:      int       ,A		:      Dict     ):






   '''simple docstring'''
   if self.remove_space:
    UpperCAmelCase__       :	Union[str, Any]        =       """ """.join(inputs.strip().split()     )
   else:
    UpperCAmelCase__       :	Tuple        =       inputs
   UpperCAmelCase__       :	List[Any]        =       outputs.replace("""``"""       ,"""\""""     ).replace("""''"""       ,"""\""""     )

   if not self.keep_accents:
    UpperCAmelCase__       :	str        =       unicodedata.normalize("""NFKD"""       ,A     )
    UpperCAmelCase__       :	List[Any]        =       """""".join([c for c in outputs if not unicodedata.combining(A     )]     )
   if self.do_lower_case:
    UpperCAmelCase__       :	List[Any]        =       outputs.lower()

   return outputs

  def  __lowercase							(	self		:      Any       ,A		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[str]        =       self.preprocess_text(A     )
   UpperCAmelCase__       :	Dict        =       self.sp_model.encode(A       ,out_type=A     )
   UpperCAmelCase__       :	Optional[Any]        =       []
   for piece in pieces:
    if len(A     ) > 1 and piece[-1] == str(""","""     ) and piece[-2].isdigit():
     UpperCAmelCase__       :	Optional[Any]        =       self.sp_model.EncodeAsPieces(piece[:-1].replace(A       ,""""""     )     )
     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
      if len(cur_pieces[0]     ) == 1:
       UpperCAmelCase__       :	str        =       cur_pieces[1:]
      else:
       UpperCAmelCase__       :	Union[str, Any]        =       cur_pieces[0][1:]
     cur_pieces.append(piece[-1]     )
     new_pieces.extend(A     )
    else:
     new_pieces.append(A     )

   return new_pieces

  def  __lowercase							(	self		:      Any       ,A		:      Optional[Any]     ):






   '''simple docstring'''
   return self.sp_model.PieceToId(A     )

  def  __lowercase							(	self		:      Tuple       ,A		:      Dict     ):






   '''simple docstring'''
   return self.sp_model.IdToPiece(A     )

  def  __lowercase							(	self		:      Tuple       ,A		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Union[str, Any]        =       """""".join(A     ).replace(A       ,""" """     ).strip()
   return out_string

  def  __lowercase							(	self		:      Tuple       ,A		:      List[int]       ,A		:      bool = False       ,A		:      bool = None       ,A		:      bool = True       ,**A		:      List[str]       ,):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       kwargs.pop("""use_source_tokenizer"""       ,A     )

   UpperCAmelCase__       :	List[Any]        =       self.convert_ids_to_tokens(A       ,skip_special_tokens=A     )

   # To avoid mixing byte-level and unicode for byte-level BPT
   # we need to build string separately for added tokens and byte-level tokens
   # cf. https://github.com/huggingface/transformers/issues/1133
   UpperCAmelCase__       :	List[Any]        =       []
   UpperCAmelCase__       :	int        =       []
   for token in filtered_tokens:
    if skip_special_tokens and token in self.all_special_ids:
     continue
    if token in self.added_tokens_encoder:
     if current_sub_text:
      sub_texts.append(self.convert_tokens_to_string(A     )     )
      UpperCAmelCase__       :	Dict        =       []
     sub_texts.append(A     )
    else:
     current_sub_text.append(A     )
   if current_sub_text:
    sub_texts.append(self.convert_tokens_to_string(A     )     )

   # Mimic the behavior of the Rust tokenizer:
   # By default, there are no spaces between special tokens
   UpperCAmelCase__       :	str        =       """""".join(A     )

   UpperCAmelCase__       :	Tuple        =       (
       clean_up_tokenization_spaces
       if clean_up_tokenization_spaces is not None
       else self.clean_up_tokenization_spaces
   )
   if clean_up_tokenization_spaces:
    UpperCAmelCase__       :	Any        =       self.clean_up_tokenization(A     )
    return clean_text
   else:
    return text

  def  __lowercase							(	self		:      Any       ,A		:      List[int]       ,A		:      Optional[List[int]] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Union[str, Any]        =       [self.sep_token_id]
   UpperCAmelCase__       :	str        =       [self.cls_token_id]
   if token_ids_a is None:
    return token_ids_a + sep + cls
   return token_ids_a + sep + token_ids_a + sep + cls

  def  __lowercase							(	self		:      str       ,A		:      List[int]       ,A		:      Optional[List[int]] = None       ,A		:      bool = False     ):






   '''simple docstring'''

   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=A       ,token_ids_a=A       ,already_has_special_tokens=A     )

   if token_ids_a is not None:
    return ([0] * len(A     )) + [1] + ([0] * len(A     )) + [1, 1]
   return ([0] * len(A     )) + [1, 1]

  def  __lowercase							(	self		:      Optional[int]       ,A		:      List[int]       ,A		:      Optional[List[int]] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       [self.sep_token_id]
   UpperCAmelCase__       :	int        =       [2]

   if token_ids_a is None:
    return len(token_ids_a + sep     ) * [0] + cls_segment_id
   return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id




  def  __lowercase							(	self		:      Union[str, Any]       ,A		:      str       ,A		:      Optional[str] = None     ):






   '''simple docstring'''
   if not os.path.isdir(A     ):
    logger.error(f"Vocabulary path ({save_directory}) should be a directory"     )
    return
   UpperCAmelCase__       :	Any        =       os.path.join(
       A       ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

   if os.path.abspath(self.vocab_file     ) != os.path.abspath(A     ) and os.path.isfile(self.vocab_file     ):
    copyfile(self.vocab_file       ,A     )
   elif not os.path.isfile(self.vocab_file     ):
    with open(A       ,"""wb"""     ) as fi:
     UpperCAmelCase__       :	str        =       self.sp_model.serialized_model_proto()
     fi.write(A     )

   return (out_vocab_file,)













"""simple docstring"""


from abc import ABC, abstractmethod
from argparse import ArgumentParser




class 						__UpperCamelCase     (     UpperCamelCase						):
					@staticmethod
					@abstractmethod
					def        UpperCAmelCase__ (							UpperCAmelCase							:	ArgumentParser   )	->       Optional[Any]:
											raise NotImplementedError()
					@abstractmethod
					def        UpperCAmelCase__ (							self							:	str   )	->       List[str]:
											raise NotImplementedError()