
"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a								=						logging.get_logger(__name__)

_a								=						"""▁"""

_a								=						{"""vocab_file""": """sentencepiece.bpe.model"""}

_a								=						{
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    }
}

_a								=						{
    """facebook/xglm-564M""": 2048,
}






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = VOCAB_FILES_NAMES
		lowercase__         = PRETRAINED_VOCAB_FILES_MAP
		lowercase__         = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase__         = ['input_ids', 'attention_mask']



		def __init__(   self							,						__a							,						__a="<s>"							,						__a="</s>"							,						__a="</s>"							,						__a="<s>"							,						__a="<unk>"							,						__a="<pad>"							,						__a = None							,						**__a							,						)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    {} if sp_model_kwargs is None else sp_model_kwargs

							# Compatibility with the original tokenizer
							_UpperCamelCase									=    7
							_UpperCamelCase									=    [F'''<madeupword{i}>''' for i in range(self.num_madeup_words)]

							_UpperCamelCase									=    kwargs.get('''additional_special_tokens'''							,						[])
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    bos_token=__a							,						eos_token=__a							,						unk_token=__a							,						sep_token=__a							,						cls_token=__a							,						pad_token=__a							,						sp_model_kwargs=self.sp_model_kwargs							,						**__a							,						)

							_UpperCamelCase									=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(str(__a))
							_UpperCamelCase									=    vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_UpperCamelCase									=    1

							# Mimic fairseq token-to-id alignment for the first 4 token
							_UpperCamelCase									=    {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							_UpperCamelCase									=    len(self.sp_model)
							_UpperCamelCase									=    {F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words)}
							self.fairseq_tokens_to_ids.update(__a)

							_UpperCamelCase									=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}



		def __getstate__(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.__dict__.copy()
							_UpperCamelCase									=    None
							_UpperCamelCase									=    self.sp_model.serialized_model_proto()
							return state



		def __setstate__(   self							,						__a)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    d

							# for backward compatibility
							if not hasattr(self							,						'''sp_model_kwargs'''):
												_UpperCamelCase									=    {}

							_UpperCamelCase									=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto)



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[int]:






							'''simple docstring'''

							if token_ids_a is None:
												return [self.sep_token_id] + token_ids_a
							_UpperCamelCase									=    [self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = False)			->			List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a							,						token_ids_a=__a							,						already_has_special_tokens=__a)

							if token_ids_a is None:
												return [1] + ([0] * len(__a))
							return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a))



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[int]:






							'''simple docstring'''
							_UpperCamelCase									=    [self.sep_token_id]

							if token_ids_a is None:
												return len(sep + token_ids_a) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a) * [0]



		@property
		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return len(self.sp_model) + self.fairseq_offset + self.num_madeup_words



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							_UpperCamelCase									=    {self.convert_ids_to_tokens(__a): i for i in range(self.vocab_size)}
							vocab.update(self.added_tokens_encoder)
							return vocab



		def 		UpperCAmelCase				(   self							,						__a)			->			List[str]:






							'''simple docstring'''
							return self.sp_model.encode(__a							,						out_type=__a)



		def 		UpperCAmelCase				(   self							,						__a)			->			Union[str, Any]:






							'''simple docstring'''
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							_UpperCamelCase									=    self.sp_model.PieceToId(__a)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def 		UpperCAmelCase				(   self							,						__a)			->			Tuple:






							'''simple docstring'''
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset)



		def 		UpperCAmelCase				(   self							,						__a)			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    ''''''.join(__a).replace(__a							,						''' ''').strip()
							return out_string




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			Tuple[str]:






							'''simple docstring'''
							if not os.path.isdir(__a):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
												return
							_UpperCamelCase									=    os.path.join(
							    __a							,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__a) and os.path.isfile(self.vocab_file):
												copyfile(self.vocab_file							,						__a)
							elif not os.path.isfile(self.vocab_file):
												with open(__a							,						'''wb''') as fi:
																	_UpperCamelCase									=    self.sp_model.serialized_model_proto()
																	fi.write(__a)

							return (out_vocab_file,)








"""simple docstring"""





from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching



def  lowerCamelCase__						(     )				->						List[str]:


					"""simple docstring"""





					import os as original_os
					from os import path as original_path
					from os import rename as original_rename
					from os.path import dirname as original_dirname
					from os.path import join as original_join

					assert _test_patching.os is original_os
					assert _test_patching.path is original_path
					assert _test_patching.join is original_join

					assert _test_patching.renamed_os is original_os
					assert _test_patching.renamed_path is original_path
					assert _test_patching.renamed_join is original_join

					_UpperCamelCase									=    '''__test_patch_submodule_mock__'''
					with patch_submodule(_test_patching,	'''os.path.join''',	__snake_case      ):
										# Every way to access os.path.join must be patched, and the rest must stay untouched

										# check os.path.join
										assert isinstance(_test_patching.os,	_PatchedModuleObj      )
										assert isinstance(_test_patching.os.path,	_PatchedModuleObj      )
										assert _test_patching.os.path.join is mock

										# check path.join
										assert isinstance(_test_patching.path,	_PatchedModuleObj      )
										assert _test_patching.path.join is mock

										# check join
										assert _test_patching.join is mock

										# check that the other attributes are untouched
										assert _test_patching.os.rename is original_rename
										assert _test_patching.path.dirname is original_dirname
										assert _test_patching.os.path.dirname is original_dirname

										# Even renamed modules or objects must be patched

										# check renamed_os.path.join
										assert isinstance(_test_patching.renamed_os,	_PatchedModuleObj      )
										assert isinstance(_test_patching.renamed_os.path,	_PatchedModuleObj      )
										assert _test_patching.renamed_os.path.join is mock

										# check renamed_path.join
										assert isinstance(_test_patching.renamed_path,	_PatchedModuleObj      )
										assert _test_patching.renamed_path.join is mock

										# check renamed_join
										assert _test_patching.renamed_join is mock

										# check that the other attributes are untouched
										assert _test_patching.renamed_os.rename is original_rename
										assert _test_patching.renamed_path.dirname is original_dirname
										assert _test_patching.renamed_os.path.dirname is original_dirname

					# check that everthing is back to normal when the patch is over

					assert _test_patching.os is original_os
					assert _test_patching.path is original_path
					assert _test_patching.join is original_join

					assert _test_patching.renamed_os is original_os
					assert _test_patching.renamed_path is original_path
					assert _test_patching.renamed_join is original_join



def  lowerCamelCase__						(     )				->						List[str]:


					"""simple docstring"""





					assert _test_patching.open is open

					_UpperCamelCase									=    '''__test_patch_submodule_builtin_mock__'''
					# _test_patching has "open" in its globals
					assert _test_patching.open is open
					with patch_submodule(_test_patching,	'''open''',	__snake_case      ):
										assert _test_patching.open is mock

					# check that everthing is back to normal when the patch is over

					assert _test_patching.open is open



def  lowerCamelCase__						(     )				->						Union[str, Any]:


					"""simple docstring"""





					_UpperCamelCase									=    '''__test_patch_submodule_missing_mock__'''
					with patch_submodule(_test_patching,	'''pandas.read_csv''',	__snake_case      ):
										pass



def  lowerCamelCase__						(     )				->						Dict:


					"""simple docstring"""





					_UpperCamelCase									=    '''__test_patch_submodule_missing_builtin_mock__'''
					# _test_patching doesn't have "len" in its globals
					assert getattr(_test_patching,	'''len''',	__snake_case      ) is None
					with patch_submodule(_test_patching,	'''len''',	__snake_case      ):
										assert _test_patching.len is mock
					assert _test_patching.len is len



def  lowerCamelCase__						(     )				->						Tuple:


					"""simple docstring"""





					_UpperCamelCase									=    '''__test_patch_submodule_start_and_stop_mock__'''
					_UpperCamelCase									=    patch_submodule(_test_patching,	'''open''',	__snake_case      )
					assert _test_patching.open is open
					patch.start()
					assert _test_patching.open is mock
					patch.stop()
					assert _test_patching.open is open



def  lowerCamelCase__						(     )				->						Optional[int]:


					"""simple docstring"""





					from os import rename as original_rename
					from os.path import dirname as original_dirname
					from os.path import join as original_join

					_UpperCamelCase									=    '''__test_patch_submodule_successive_join__'''
					_UpperCamelCase									=    '''__test_patch_submodule_successive_dirname__'''
					_UpperCamelCase									=    '''__test_patch_submodule_successive_rename__'''
					assert _test_patching.os.path.join is original_join
					assert _test_patching.os.path.dirname is original_dirname
					assert _test_patching.os.rename is original_rename

					with patch_submodule(_test_patching,	'''os.path.join''',	__snake_case      ):
										with patch_submodule(_test_patching,	'''os.rename''',	__snake_case      ):
															with patch_submodule(_test_patching,	'''os.path.dirname''',	__snake_case      ):
																				assert _test_patching.os.path.join is mock_join
																				assert _test_patching.os.path.dirname is mock_dirname
																				assert _test_patching.os.rename is mock_rename

    # try another order
					with patch_submodule(_test_patching,	'''os.rename''',	__snake_case      ):
										with patch_submodule(_test_patching,	'''os.path.join''',	__snake_case      ):
															with patch_submodule(_test_patching,	'''os.path.dirname''',	__snake_case      ):
																				assert _test_patching.os.path.join is mock_join
																				assert _test_patching.os.path.dirname is mock_dirname
																				assert _test_patching.os.rename is mock_rename

					assert _test_patching.os.path.join is original_join
					assert _test_patching.os.path.dirname is original_dirname
					assert _test_patching.os.rename is original_rename



def  lowerCamelCase__						(     )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    '''__test_patch_submodule_doesnt_exist_mock__'''
					with patch_submodule(_test_patching,	'''__module_that_doesn_exist__.__attribute_that_doesn_exist__''',	__snake_case      ):
										pass
					with patch_submodule(_test_patching,	'''os.__attribute_that_doesn_exist__''',	__snake_case      ):
										pass






