




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCAmelCase_						 =					logging.get_logger(__name__)

lowerCAmelCase_						 =					{'vocab_file': 'spiece.model'}

lowerCAmelCase_						 =					{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

lowerCAmelCase_						 =					{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
lowerCAmelCase_						 =					0
lowerCAmelCase_						 =					1
lowerCAmelCase_						 =					2
lowerCAmelCase_						 =					3
lowerCAmelCase_						 =					4
class 		_A   (    _lowerCamelCase ):
  _UpperCamelCase  :     Optional[int]					=     VOCAB_FILES_NAMES
  _UpperCamelCase  :     Dict					=     PRETRAINED_VOCAB_FILES_MAP
  _UpperCamelCase  :     Optional[int]					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCamelCase  :     Optional[int]					=     'left'






  def __init__(    self						:      Optional[Any] ,		_A						:      Optional[int] ,		_A						:      List[str]=False ,		_A						:      Optional[int]=True ,		_A						:      str=False ,		_A						:      Tuple="<s>" ,		_A						:      Any="</s>" ,		_A						:      Optional[Any]="<unk>" ,		_A						:      Dict="<sep>" ,		_A						:      str="<pad>" ,		_A						:      int="<cls>" ,		_A						:      Union[str, Any]="<mask>" ,		_A						:      Tuple=["<eop>", "<eod>"] ,		_A						:      str = None ,		**_A						:      List[str] ,		)						->       List[Any]:






       """simple docstring"""







       lowercase   :  Union[str, Any]        =					AddedToken(_A ,		lstrip=_A ,		rstrip=_A ) if isinstance(_A ,		_A ) else mask_token

       lowercase   :  Optional[Any]        =					{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=_A ,		remove_space=_A ,		keep_accents=_A ,		bos_token=_A ,		eos_token=_A ,		unk_token=_A ,		sep_token=_A ,		pad_token=_A ,		cls_token=_A ,		mask_token=_A ,		additional_special_tokens=_A ,		sp_model_kwargs=self.sp_model_kwargs ,		**_A ,		)

       lowercase   :  Optional[Any]        =					3

       lowercase   :  Optional[Any]        =					do_lower_case
       lowercase   :  int        =					remove_space
       lowercase   :  str        =					keep_accents
       lowercase   :  List[str]        =					vocab_file

       lowercase   :  str        =					spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(_A )






  @property
  def    __a     (    self						:      List[str] )						->       Optional[int]:






       """simple docstring"""







       return len(self.sp_model )






  def    __a     (    self						:      Union[str, Any] )						->       List[str]:






       """simple docstring"""







       lowercase   :  Optional[int]        =					{self.convert_ids_to_tokens(_A ): i for i in range(self.vocab_size )}
       vocab.update(self.added_tokens_encoder )
       return vocab






  def __getstate__(    self						:      Optional[int] )						->       Tuple:






       """simple docstring"""







       lowercase   :  Any        =					self.__dict__.copy()
       lowercase   :  Optional[Any]        =					None
       return state






  def __setstate__(    self						:      List[Any] ,		_A						:      Optional[Any] )						->       Union[str, Any]:






       """simple docstring"""







       lowercase   :  int        =					d

       # for backward compatibility
       if not hasattr(self ,		'''sp_model_kwargs''' ):
            lowercase   :  Union[str, Any]        =					{}

       lowercase   :  Dict        =					spm.SentencePieceProcessor(**self.sp_model_kwargs )
       self.sp_model.Load(self.vocab_file )






  def    __a     (    self						:      Union[str, Any] ,		_A						:      Any )						->       List[Any]:






       """simple docstring"""







       if self.remove_space:
            lowercase   :  Optional[int]        =					''' '''.join(inputs.strip().split() )
       else:
            lowercase   :  str        =					inputs
       lowercase   :  Dict        =					outputs.replace('''``''' ,		'''\"''' ).replace('''\'\'''' ,		'''\"''' )

       if not self.keep_accents:
            lowercase   :  int        =					unicodedata.normalize('''NFKD''' ,		_A )
            lowercase   :  Tuple        =					''''''.join([c for c in outputs if not unicodedata.combining(_A )] )
       if self.do_lower_case:
            lowercase   :  Optional[Any]        =					outputs.lower()

       return outputs






  def    __a     (    self						:      List[Any] ,		_A						:      List[str] )						->       Union[str, Any]:






       """simple docstring"""







       lowercase   :  str        =					self.preprocess_text(_A )
       lowercase   :  Optional[Any]        =					self.sp_model.encode(_A ,		out_type=_A )
       lowercase   :  List[Any]        =					[]
       for piece in pieces:
            if len(_A ) > 1 and piece[-1] == str(''',''' ) and piece[-2].isdigit():
                 lowercase   :  Union[str, Any]        =					self.sp_model.EncodeAsPieces(piece[:-1].replace(_A ,		'''''' ) )
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0] ) == 1:
                           lowercase   :  Any        =					cur_pieces[1:]
                      else:
                           lowercase   :  Union[str, Any]        =					cur_pieces[0][1:]
                 cur_pieces.append(piece[-1] )
                 new_pieces.extend(_A )
            else:
                 new_pieces.append(_A )

       return new_pieces






  def    __a     (    self						:      Optional[Any] ,		_A						:      int )						->       int:






       """simple docstring"""







       return self.sp_model.PieceToId(_A )






  def    __a     (    self						:      List[Any] ,		_A						:      Union[str, Any] )						->       int:






       """simple docstring"""







       return self.sp_model.IdToPiece(_A )






  def    __a     (    self						:      Dict ,		_A						:      Dict )						->       Optional[Any]:






       """simple docstring"""







       lowercase   :  Optional[Any]        =					''''''.join(_A ).replace(_A ,		''' ''' ).strip()
       return out_string






  def    __a     (    self						:      str ,		_A						:      Dict ,		_A						:      Optional[int] = False ,		_A						:      Union[str, Any] = None ,		_A						:      Dict = True ,		**_A						:      int ,		)						->       Optional[int]:






       """simple docstring"""







       lowercase   :  Any        =					kwargs.pop('''use_source_tokenizer''' ,		_A )

       lowercase   :  List[Any]        =					self.convert_ids_to_tokens(_A ,		skip_special_tokens=_A )

       # To avoid mixing byte-level and unicode for byte-level BPT
       # we need to build string separately for added tokens and byte-level tokens
       # cf. https://github.com/huggingface/transformers/issues/1133
       lowercase   :  Union[str, Any]        =					[]
       lowercase   :  Any        =					[]
       for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                 continue
            if token in self.added_tokens_encoder:
                 if current_sub_text:
                      sub_texts.append(self.convert_tokens_to_string(_A ) )
                      lowercase   :  int        =					[]
                 sub_texts.append(_A )
            else:
                 current_sub_text.append(_A )
       if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(_A ) )

       # Mimic the behavior of the Rust tokenizer:
       # By default, there are no spaces between special tokens
       lowercase   :  Any        =					''''''.join(_A )

       lowercase   :  List[str]        =					(
           clean_up_tokenization_spaces
           if clean_up_tokenization_spaces is not None
           else self.clean_up_tokenization_spaces
       )
       if clean_up_tokenization_spaces:
            lowercase   :  List[Any]        =					self.clean_up_tokenization(_A )
            return clean_text
       else:
            return text






  def    __a     (    self						:      Optional[Any] ,		_A						:      Tuple ,		_A						:      Union[str, Any] = None )						->       str:






       """simple docstring"""







       lowercase   :  List[Any]        =					[self.sep_token_id]
       lowercase   :  Union[str, Any]        =					[self.cls_token_id]
       if token_ids_a is None:
            return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls






  def    __a     (    self						:      int ,		_A						:      List[str] ,		_A						:      Any = None ,		_A						:      str = False )						->       Any:






       """simple docstring"""







       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=_A ,		token_ids_a=_A ,		already_has_special_tokens=_A )

       if token_ids_a is not None:
            return ([0] * len(_A )) + [1] + ([0] * len(_A )) + [1, 1]
       return ([0] * len(_A )) + [1, 1]






  def    __a     (    self						:      Optional[int] ,		_A						:      List[str] ,		_A						:      int = None )						->       Optional[Any]:






       """simple docstring"""







       lowercase   :  int        =					[self.sep_token_id]
       lowercase   :  List[str]        =					[2]

       if token_ids_a is None:
            return len(token_ids_a + sep ) * [0] + cls_segment_id
       return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id






  def    __a     (    self						:      Tuple ,		_A						:      List[str] ,		_A						:      List[Any] = None )						->       str:






       """simple docstring"""







       if not os.path.isdir(_A ):
            logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
            return
       lowercase   :  Any        =					os.path.join(
           _A ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

       if os.path.abspath(self.vocab_file ) != os.path.abspath(_A ) and os.path.isfile(self.vocab_file ):
            copyfile(self.vocab_file ,		_A )
       elif not os.path.isfile(self.vocab_file ):
            with open(_A ,		'''wb''' ) as fi:
                 lowercase   :  List[str]        =					self.sp_model.serialized_model_proto()
                 fi.write(_A )

       return (out_vocab_file,)



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__UpperCamelCase     		=							{
    'configuration_xlm_roberta_xl': [
        'XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XLMRobertaXLConfig',
        'XLMRobertaXLOnnxConfig',
    ],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__UpperCamelCase     		=							[
						    'XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'XLMRobertaXLForCausalLM',
						    'XLMRobertaXLForMaskedLM',
						    'XLMRobertaXLForMultipleChoice',
						    'XLMRobertaXLForQuestionAnswering',
						    'XLMRobertaXLForSequenceClassification',
						    'XLMRobertaXLForTokenClassification',
						    'XLMRobertaXLModel',
						    'XLMRobertaXLPreTrainedModel',
						]

if TYPE_CHECKING:
						from .configuration_xlm_roberta_xl import (
						    XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    XLMRobertaXLConfig,
						    XLMRobertaXLOnnxConfig,
						)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_xlm_roberta_xl import (
												    XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
												    XLMRobertaXLForCausalLM,
												    XLMRobertaXLForMaskedLM,
												    XLMRobertaXLForMultipleChoice,
												    XLMRobertaXLForQuestionAnswering,
												    XLMRobertaXLForSequenceClassification,
												    XLMRobertaXLForTokenClassification,
												    XLMRobertaXLModel,
												    XLMRobertaXLPreTrainedModel,
												)

else:
						import sys

						__UpperCamelCase     		=							_LazyModule(__name__, globals()['__file__'], _import_structure)


