





'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE       =     logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE       =     '▁'

_SCREAMING_SNAKE_CASE       =     {'vocab_file': 'sentencepiece.bpe.model'}

_SCREAMING_SNAKE_CASE       =     {
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

_SCREAMING_SNAKE_CASE       =     {
    'facebook/xglm-564M': 2_0_4_8,
}






class 				__lowercase							(   __SCREAMING_SNAKE_CASE    ):







	'''simple docstring'''

	a     :					Optional[int]           =							VOCAB_FILES_NAMES
	a     :					Any           =							PRETRAINED_VOCAB_FILES_MAP
	a     :					Any           =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a     :					Dict           =							["input_ids", "attention_mask"]


	def __init__(self  ,_lowerCamelCase  ,_lowerCamelCase="<s>"  ,_lowerCamelCase="</s>"  ,_lowerCamelCase="</s>"  ,_lowerCamelCase="<s>"  ,_lowerCamelCase="<unk>"  ,_lowerCamelCase="<pad>"  ,_lowerCamelCase = None  ,**_lowerCamelCase  ,)				->				None:







				'''simple docstring'''





				__lowercase					    =				{} if sp_model_kwargs is None else sp_model_kwargs

				# Compatibility with the original tokenizer
				__lowercase					    =				7
				__lowercase					    =				[f"<madeupword{i}>" for i in range(self.num_madeup_words			)]

				__lowercase					    =				kwargs.get('''additional_special_tokens'''  ,[]			)
				kwargs["additional_special_tokens"] += [
				    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    bos_token=_SCREAMING_SNAKE_CASE  ,eos_token=_SCREAMING_SNAKE_CASE  ,unk_token=_SCREAMING_SNAKE_CASE  ,sep_token=_SCREAMING_SNAKE_CASE  ,cls_token=_SCREAMING_SNAKE_CASE  ,pad_token=_SCREAMING_SNAKE_CASE  ,sp_model_kwargs=self.sp_model_kwargs  ,**_SCREAMING_SNAKE_CASE  ,)

				__lowercase					    =				spm.SentencePieceProcessor(**self.sp_model_kwargs			)
				self.sp_model.Load(str(_SCREAMING_SNAKE_CASE			)			)
				__lowercase					    =				vocab_file

				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
				# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
				# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				__lowercase					    =				1

				# Mimic fairseq token-to-id alignment for the first 4 token
				__lowercase					    =				{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

				__lowercase					    =				len(self.sp_model			)
				__lowercase					    =				{f"<madeupword{i}>": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words			)}
				self.fairseq_tokens_to_ids.update(_SCREAMING_SNAKE_CASE			)

				__lowercase					    =				{v: k for k, v in self.fairseq_tokens_to_ids.items()}


	def __getstate__(self			)				->				Union[str, Any]:







				'''simple docstring'''





				__lowercase					    =				self.__dict__.copy()
				__lowercase					    =				None
				__lowercase					    =				self.sp_model.serialized_model_proto()
				return state


	def __setstate__(self  ,_lowerCamelCase			)				->				Any:







				'''simple docstring'''





				__lowercase					    =				d

				# for backward compatibility
				if not hasattr(self  ,'''sp_model_kwargs'''			):
							__lowercase					    =				{}

				__lowercase					    =				spm.SentencePieceProcessor(**self.sp_model_kwargs			)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto			)


	def     _UpperCAmelCase  (self  ,_lowerCamelCase  ,_lowerCamelCase = None			)				->				List[int]:







				'''simple docstring'''





				if token_ids_a is None:
							return [self.sep_token_id] + token_ids_a
				__lowercase					    =				[self.sep_token_id]
				return sep + token_ids_a + sep + sep + token_ids_a


	def     _UpperCAmelCase  (self  ,_lowerCamelCase  ,_lowerCamelCase = None  ,_lowerCamelCase = False			)				->				List[int]:







				'''simple docstring'''





				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_SCREAMING_SNAKE_CASE  ,token_ids_a=_SCREAMING_SNAKE_CASE  ,already_has_special_tokens=_SCREAMING_SNAKE_CASE			)

				if token_ids_a is None:
							return [1] + ([0] * len(_SCREAMING_SNAKE_CASE			))
				return [1] + ([0] * len(_SCREAMING_SNAKE_CASE			)) + [1, 1] + ([0] * len(_SCREAMING_SNAKE_CASE			))


	def     _UpperCAmelCase  (self  ,_lowerCamelCase  ,_lowerCamelCase = None			)				->				List[int]:







				'''simple docstring'''





				__lowercase					    =				[self.sep_token_id]

				if token_ids_a is None:
							return len(sep + token_ids_a			) * [0]
				return len(sep + token_ids_a + sep + sep + token_ids_a			) * [0]


	@property
	def     _UpperCAmelCase  (self			)				->				str:







				'''simple docstring'''





				return len(self.sp_model			) + self.fairseq_offset + self.num_madeup_words


	def     _UpperCAmelCase  (self			)				->				Optional[int]:







				'''simple docstring'''





				__lowercase					    =				{self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE			): i for i in range(self.vocab_size			)}
				vocab.update(self.added_tokens_encoder			)
				return vocab


	def     _UpperCAmelCase  (self  ,_lowerCamelCase			)				->				List[str]:







				'''simple docstring'''





				return self.sp_model.encode(_SCREAMING_SNAKE_CASE  ,out_type=_SCREAMING_SNAKE_CASE			)


	def     _UpperCAmelCase  (self  ,_lowerCamelCase			)				->				Tuple:







				'''simple docstring'''





				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				__lowercase					    =				self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE			)

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


	def     _UpperCAmelCase  (self  ,_lowerCamelCase			)				->				Union[str, Any]:







				'''simple docstring'''





				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset			)


	def     _UpperCAmelCase  (self  ,_lowerCamelCase			)				->				str:







				'''simple docstring'''





				__lowercase					    =				"".join(_SCREAMING_SNAKE_CASE			).replace(_SCREAMING_SNAKE_CASE  ,''' '''			).strip()
				return out_string


	def     _UpperCAmelCase  (self  ,_lowerCamelCase  ,_lowerCamelCase = None			)				->				Tuple[str]:







				'''simple docstring'''





				if not os.path.isdir(_SCREAMING_SNAKE_CASE			):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
							return
				__lowercase					    =				os.path.join(
				    _SCREAMING_SNAKE_CASE  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

				if os.path.abspath(self.vocab_file			) != os.path.abspath(_SCREAMING_SNAKE_CASE			) and os.path.isfile(self.vocab_file			):
							copyfile(self.vocab_file  ,_SCREAMING_SNAKE_CASE			)
				elif not os.path.isfile(self.vocab_file			):
							with open(_SCREAMING_SNAKE_CASE  ,'''wb'''			) as fi:
										__lowercase					    =				self.sp_model.serialized_model_proto()
										fi.write(_SCREAMING_SNAKE_CASE			)

				return (out_vocab_file,)










"""simple docstring"""






from ..utils import DummyObject, requires_backends



class      A__  (    metaclass=__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      ['torch', 'transformers', 'onnx']

	def __init__(	self:						Union[str, Any]			,     *_SCREAMING_SNAKE_CASE:						Any			,     **_SCREAMING_SNAKE_CASE:						int)							->			List[Any]:

				"""simple docstring"""

				requires_backends(self			,     ["torch", "transformers", "onnx"])

	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Union[str, Any]			,     *_SCREAMING_SNAKE_CASE:						List[Any]			,     **_SCREAMING_SNAKE_CASE:						Union[str, Any])							->			Union[str, Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						int			,     *_SCREAMING_SNAKE_CASE:						Any			,     **_SCREAMING_SNAKE_CASE:						str)							->			Union[str, Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])






class      A__  (    metaclass=__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      ['torch', 'transformers', 'onnx']

	def __init__(	self:						Tuple			,     *_SCREAMING_SNAKE_CASE:						Optional[Any]			,     **_SCREAMING_SNAKE_CASE:						List[str])							->			Optional[int]:

				"""simple docstring"""

				requires_backends(self			,     ["torch", "transformers", "onnx"])

	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Dict			,     *_SCREAMING_SNAKE_CASE:						Optional[Any]			,     **_SCREAMING_SNAKE_CASE:						int)							->			List[Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Optional[Any]			,     *_SCREAMING_SNAKE_CASE:						Dict			,     **_SCREAMING_SNAKE_CASE:						Optional[Any])							->			Union[str, Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])






class      A__  (    metaclass=__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      ['torch', 'transformers', 'onnx']

	def __init__(	self:						int			,     *_SCREAMING_SNAKE_CASE:						List[Any]			,     **_SCREAMING_SNAKE_CASE:						int)							->			List[str]:

				"""simple docstring"""

				requires_backends(self			,     ["torch", "transformers", "onnx"])

	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Dict			,     *_SCREAMING_SNAKE_CASE:						Any			,     **_SCREAMING_SNAKE_CASE:						Any)							->			str:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						List[Any]			,     *_SCREAMING_SNAKE_CASE:						List[str]			,     **_SCREAMING_SNAKE_CASE:						int)							->			int:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])






class      A__  (    metaclass=__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      ['torch', 'transformers', 'onnx']

	def __init__(	self:						Tuple			,     *_SCREAMING_SNAKE_CASE:						List[str]			,     **_SCREAMING_SNAKE_CASE:						List[str])							->			Optional[Any]:

				"""simple docstring"""

				requires_backends(self			,     ["torch", "transformers", "onnx"])

	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						List[str]			,     *_SCREAMING_SNAKE_CASE:						Tuple			,     **_SCREAMING_SNAKE_CASE:						int)							->			int:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Optional[int]			,     *_SCREAMING_SNAKE_CASE:						str			,     **_SCREAMING_SNAKE_CASE:						Any)							->			int:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])






class      A__  (    metaclass=__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      ['torch', 'transformers', 'onnx']

	def __init__(	self:						Optional[Any]			,     *_SCREAMING_SNAKE_CASE:						Union[str, Any]			,     **_SCREAMING_SNAKE_CASE:						Optional[int])							->			str:

				"""simple docstring"""

				requires_backends(self			,     ["torch", "transformers", "onnx"])

	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Optional[int]			,     *_SCREAMING_SNAKE_CASE:						List[Any]			,     **_SCREAMING_SNAKE_CASE:						Tuple)							->			Union[str, Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						int			,     *_SCREAMING_SNAKE_CASE:						str			,     **_SCREAMING_SNAKE_CASE:						Any)							->			Optional[Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])






class      A__  (    metaclass=__SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      ['torch', 'transformers', 'onnx']

	def __init__(	self:						List[Any]			,     *_SCREAMING_SNAKE_CASE:						List[str]			,     **_SCREAMING_SNAKE_CASE:						List[str])							->			int:

				"""simple docstring"""

				requires_backends(self			,     ["torch", "transformers", "onnx"])

	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Union[str, Any]			,     *_SCREAMING_SNAKE_CASE:						Optional[int]			,     **_SCREAMING_SNAKE_CASE:						Optional[Any])							->			List[Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						str			,     *_SCREAMING_SNAKE_CASE:						Optional[int]			,     **_SCREAMING_SNAKE_CASE:						Dict)							->			Optional[Any]:

				"""simple docstring"""

				requires_backends(cls			,     ["torch", "transformers", "onnx"])