



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_snake_case				=					logging.get_logger(__name__)

_snake_case				=					{'vocab_file': 'spiece.model'}

_snake_case				=					{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

_snake_case				=					{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
_snake_case				=					0
_snake_case				=					1
_snake_case				=					2
_snake_case				=					3
_snake_case				=					4






class 		lowerCAmelCase_ (			_a ):







		"""simple docstring"""
		UpperCAmelCase__													=       VOCAB_FILES_NAMES
		UpperCAmelCase__													=       PRETRAINED_VOCAB_FILES_MAP
		UpperCAmelCase__													=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCAmelCase__													=       """left"""
		def __init__(		self     ,    _SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE=False     ,    _SCREAMING_SNAKE_CASE=True     ,    _SCREAMING_SNAKE_CASE=False     ,    _SCREAMING_SNAKE_CASE="<s>"     ,    _SCREAMING_SNAKE_CASE="</s>"     ,    _SCREAMING_SNAKE_CASE="<unk>"     ,    _SCREAMING_SNAKE_CASE="<sep>"     ,    _SCREAMING_SNAKE_CASE="<pad>"     ,    _SCREAMING_SNAKE_CASE="<cls>"     ,    _SCREAMING_SNAKE_CASE="<mask>"     ,    _SCREAMING_SNAKE_CASE=["<eop>", "<eod>"]     ,    _SCREAMING_SNAKE_CASE = None     ,    **_SCREAMING_SNAKE_CASE     ,    )     ->  None:
									__UpperCamelCase          =  AddedToken(_SCREAMING_SNAKE_CASE     ,    lstrip=_SCREAMING_SNAKE_CASE     ,    rstrip=_SCREAMING_SNAKE_CASE			) if isinstance(_SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE			) else mask_token

									__UpperCamelCase          =  {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_SCREAMING_SNAKE_CASE     ,    remove_space=_SCREAMING_SNAKE_CASE     ,    keep_accents=_SCREAMING_SNAKE_CASE     ,    bos_token=_SCREAMING_SNAKE_CASE     ,    eos_token=_SCREAMING_SNAKE_CASE     ,    unk_token=_SCREAMING_SNAKE_CASE     ,    sep_token=_SCREAMING_SNAKE_CASE     ,    pad_token=_SCREAMING_SNAKE_CASE     ,    cls_token=_SCREAMING_SNAKE_CASE     ,    mask_token=_SCREAMING_SNAKE_CASE     ,    additional_special_tokens=_SCREAMING_SNAKE_CASE     ,    sp_model_kwargs=self.sp_model_kwargs     ,    **_SCREAMING_SNAKE_CASE     ,    )

									__UpperCamelCase          =  3

									__UpperCamelCase          =  do_lower_case
									__UpperCamelCase          =  remove_space
									__UpperCamelCase          =  keep_accents
									__UpperCamelCase          =  vocab_file

									__UpperCamelCase          =  spm.SentencePieceProcessor(**self.sp_model_kwargs			)
									self.sp_model.Load(_SCREAMING_SNAKE_CASE			)
		@property
		def 						__lowercase(		self			)     ->  Union[str, Any]:
									return len(self.sp_model			)
		def 						__lowercase(		self			)     ->  Tuple:
									__UpperCamelCase          =  {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE			): i for i in range(self.vocab_size			)}
									vocab.update(self.added_tokens_encoder			)
									return vocab
		def __getstate__(		self			)     ->  str:
									__UpperCamelCase          =  self.__dict__.copy()
									__UpperCamelCase          =  None
									return state
		def __setstate__(		self     ,    _SCREAMING_SNAKE_CASE			)     ->  int:
									__UpperCamelCase          =  d

									# for backward compatibility
									if not hasattr(self     ,    'sp_model_kwargs'			):
																__UpperCamelCase          =  {}

									__UpperCamelCase          =  spm.SentencePieceProcessor(**self.sp_model_kwargs			)
									self.sp_model.Load(self.vocab_file			)
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE			)     ->  int:

									if self.remove_space:
																__UpperCamelCase          =  ' '.join(inputs.strip().split()			)
									else:
																__UpperCamelCase          =  inputs
									__UpperCamelCase          =  outputs.replace('``'     ,    '\"'			).replace('\'\''     ,    '\"'			)

									if not self.keep_accents:
																__UpperCamelCase          =  unicodedata.normalize('NFKD'     ,    _SCREAMING_SNAKE_CASE			)
																__UpperCamelCase          =  ''.join([c for c in outputs if not unicodedata.combining(_SCREAMING_SNAKE_CASE			)]			)
									if self.do_lower_case:
																__UpperCamelCase          =  outputs.lower()

									return outputs
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE			)     ->  List[str]:
									__UpperCamelCase          =  self.preprocess_text(_SCREAMING_SNAKE_CASE			)
									__UpperCamelCase          =  self.sp_model.encode(_SCREAMING_SNAKE_CASE     ,    out_type=_SCREAMING_SNAKE_CASE			)
									__UpperCamelCase          =  []
									for piece in pieces:
																if len(_SCREAMING_SNAKE_CASE			) > 1 and piece[-1] == str(','			) and piece[-2].isdigit():
																							__UpperCamelCase          =  self.sp_model.EncodeAsPieces(piece[:-1].replace(_SCREAMING_SNAKE_CASE     ,    ''			)			)
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																														if len(cur_pieces[0]			) == 1:
																																					__UpperCamelCase          =  cur_pieces[1:]
																														else:
																																					__UpperCamelCase          =  cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]			)
																							new_pieces.extend(_SCREAMING_SNAKE_CASE			)
																else:
																							new_pieces.append(_SCREAMING_SNAKE_CASE			)

									return new_pieces
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE			)     ->  Optional[Any]:
									return self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE			)
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE			)     ->  Optional[int]:
									return self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE			)
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE			)     ->  List[Any]:
									__UpperCamelCase          =  ''.join(_SCREAMING_SNAKE_CASE			).replace(_SCREAMING_SNAKE_CASE     ,    ' '			).strip()
									return out_string
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE = False     ,    _SCREAMING_SNAKE_CASE = None     ,    _SCREAMING_SNAKE_CASE = True     ,    **_SCREAMING_SNAKE_CASE     ,    )     ->  str:
									__UpperCamelCase          =  kwargs.pop('use_source_tokenizer'     ,    _SCREAMING_SNAKE_CASE			)

									__UpperCamelCase          =  self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE     ,    skip_special_tokens=_SCREAMING_SNAKE_CASE			)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									__UpperCamelCase          =  []
									__UpperCamelCase          =  []
									for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																							continue
																if token in self.added_tokens_encoder:
																							if current_sub_text:
																														sub_texts.append(self.convert_tokens_to_string(_SCREAMING_SNAKE_CASE			)			)
																														__UpperCamelCase          =  []
																							sub_texts.append(_SCREAMING_SNAKE_CASE			)
																else:
																							current_sub_text.append(_SCREAMING_SNAKE_CASE			)
									if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(_SCREAMING_SNAKE_CASE			)			)

									# Mimic the behavior of the Rust tokenizer:
									# By default, there are no spaces between special tokens
									__UpperCamelCase          =  ''.join(_SCREAMING_SNAKE_CASE			)

									__UpperCamelCase          =  (
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
																__UpperCamelCase          =  self.clean_up_tokenization(_SCREAMING_SNAKE_CASE			)
																return clean_text
									else:
																return text
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE = None			)     ->  List[int]:
									__UpperCamelCase          =  [self.sep_token_id]
									__UpperCamelCase          =  [self.cls_token_id]
									if token_ids_a is None:
																return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE = None     ,    _SCREAMING_SNAKE_CASE = False			)     ->  List[int]:

									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_SCREAMING_SNAKE_CASE     ,    token_ids_a=_SCREAMING_SNAKE_CASE     ,    already_has_special_tokens=_SCREAMING_SNAKE_CASE			)

									if token_ids_a is not None:
																return ([0] * len(_SCREAMING_SNAKE_CASE			)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE			)) + [1, 1]
									return ([0] * len(_SCREAMING_SNAKE_CASE			)) + [1, 1]
		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE = None			)     ->  List[int]:
									__UpperCamelCase          =  [self.sep_token_id]
									__UpperCamelCase          =  [2]

									if token_ids_a is None:
																return len(token_ids_a + sep			) * [0] + cls_segment_id
									return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id



		def 						__lowercase(		self     ,    _SCREAMING_SNAKE_CASE     ,    _SCREAMING_SNAKE_CASE = None			)     ->  Tuple[str]:

									if not os.path.isdir(_SCREAMING_SNAKE_CASE			):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
																return
									__UpperCamelCase          =  os.path.join(
									    _SCREAMING_SNAKE_CASE     ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

									if os.path.abspath(self.vocab_file			) != os.path.abspath(_SCREAMING_SNAKE_CASE			) and os.path.isfile(self.vocab_file			):
																copyfile(self.vocab_file     ,    _SCREAMING_SNAKE_CASE			)
									elif not os.path.isfile(self.vocab_file			):
																with open(_SCREAMING_SNAKE_CASE     ,    'wb'			) as fi:
																							__UpperCamelCase          =  self.sp_model.serialized_model_proto()
																							fi.write(_SCREAMING_SNAKE_CASE			)

									return (out_vocab_file,)







from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching





def 						_lowerCAmelCase							(     )    ->						Tuple:


						'''simple docstring'''






						import os as original_os
						from os import path as original_path
						from os import rename as original_rename
						from os.path import dirname as original_dirname
						from os.path import join as original_join

						assert _test_patching.os is original_os
						assert _test_patching.path is original_path
						assert _test_patching.join is original_join

						assert _test_patching.renamed_os is original_os
						assert _test_patching.renamed_path is original_path
						assert _test_patching.renamed_join is original_join

						__snake_case				=		"__test_patch_submodule_mock__"
						with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
												# Every way to access os.path.join must be patched, and the rest must stay untouched

												# check os.path.join
												assert isinstance(_test_patching.os   ,						_PatchedModuleObj			)
												assert isinstance(_test_patching.os.path   ,						_PatchedModuleObj			)
												assert _test_patching.os.path.join is mock

												# check path.join
												assert isinstance(_test_patching.path   ,						_PatchedModuleObj			)
												assert _test_patching.path.join is mock

												# check join
												assert _test_patching.join is mock

												# check that the other attributes are untouched
												assert _test_patching.os.rename is original_rename
												assert _test_patching.path.dirname is original_dirname
												assert _test_patching.os.path.dirname is original_dirname

												# Even renamed modules or objects must be patched

												# check renamed_os.path.join
												assert isinstance(_test_patching.renamed_os   ,						_PatchedModuleObj			)
												assert isinstance(_test_patching.renamed_os.path   ,						_PatchedModuleObj			)
												assert _test_patching.renamed_os.path.join is mock

												# check renamed_path.join
												assert isinstance(_test_patching.renamed_path   ,						_PatchedModuleObj			)
												assert _test_patching.renamed_path.join is mock

												# check renamed_join
												assert _test_patching.renamed_join is mock

												# check that the other attributes are untouched
												assert _test_patching.renamed_os.rename is original_rename
												assert _test_patching.renamed_path.dirname is original_dirname
												assert _test_patching.renamed_os.path.dirname is original_dirname

						# check that everthing is back to normal when the patch is over

						assert _test_patching.os is original_os
						assert _test_patching.path is original_path
						assert _test_patching.join is original_join

						assert _test_patching.renamed_os is original_os
						assert _test_patching.renamed_path is original_path
						assert _test_patching.renamed_join is original_join





def 						_lowerCAmelCase							(     )    ->						int:


						'''simple docstring'''






						assert _test_patching.open is open

						__snake_case				=		"__test_patch_submodule_builtin_mock__"
						# _test_patching has "open" in its globals
						assert _test_patching.open is open
						with patch_submodule(_test_patching   ,						"open"   ,						_lowerCAmelCase			):
												assert _test_patching.open is mock

						# check that everthing is back to normal when the patch is over

						assert _test_patching.open is open





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_missing_mock__"
						with patch_submodule(_test_patching   ,						"pandas.read_csv"   ,						_lowerCAmelCase			):
												pass





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_missing_builtin_mock__"
						# _test_patching doesn't have "len" in its globals
						assert getattr(_test_patching   ,						"len"   ,						_lowerCAmelCase			) is None
						with patch_submodule(_test_patching   ,						"len"   ,						_lowerCAmelCase			):
												assert _test_patching.len is mock
						assert _test_patching.len is len





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_start_and_stop_mock__"
						__snake_case				=		patch_submodule(_test_patching   ,						"open"   ,						_lowerCAmelCase			)
						assert _test_patching.open is open
						patch.start()
						assert _test_patching.open is mock
						patch.stop()
						assert _test_patching.open is open





def 						_lowerCAmelCase							(     )    ->						Any:


						'''simple docstring'''






						from os import rename as original_rename
						from os.path import dirname as original_dirname
						from os.path import join as original_join

						__snake_case				=		"__test_patch_submodule_successive_join__"
						__snake_case				=		"__test_patch_submodule_successive_dirname__"
						__snake_case				=		"__test_patch_submodule_successive_rename__"
						assert _test_patching.os.path.join is original_join
						assert _test_patching.os.path.dirname is original_dirname
						assert _test_patching.os.rename is original_rename

						with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
												with patch_submodule(_test_patching   ,						"os.rename"   ,						_lowerCAmelCase			):
																		with patch_submodule(_test_patching   ,						"os.path.dirname"   ,						_lowerCAmelCase			):
																								assert _test_patching.os.path.join is mock_join
																								assert _test_patching.os.path.dirname is mock_dirname
																								assert _test_patching.os.rename is mock_rename

    # try another order
						with patch_submodule(_test_patching   ,						"os.rename"   ,						_lowerCAmelCase			):
												with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
																		with patch_submodule(_test_patching   ,						"os.path.dirname"   ,						_lowerCAmelCase			):
																								assert _test_patching.os.path.join is mock_join
																								assert _test_patching.os.path.dirname is mock_dirname
																								assert _test_patching.os.rename is mock_rename

						assert _test_patching.os.path.join is original_join
						assert _test_patching.os.path.dirname is original_dirname
						assert _test_patching.os.rename is original_rename





def 						_lowerCAmelCase							(     )    ->						str:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_doesnt_exist_mock__"
						with patch_submodule(_test_patching   ,						"__module_that_doesn_exist__.__attribute_that_doesn_exist__"   ,						_lowerCAmelCase			):
												pass
						with patch_submodule(_test_patching   ,						"os.__attribute_that_doesn_exist__"   ,						_lowerCAmelCase			):
												pass





