





'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__magic_name__   :						Optional[int]    =			logging.get_logger(__name__)

__magic_name__   :						str    =			"""▁"""

__magic_name__   :						int    =			{"""vocab_file""": """sentencepiece.bpe.model"""}

__magic_name__   :						List[str]    =			{
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    }
}

__magic_name__   :						Union[str, Any]    =			{
    """facebook/xglm-564M""": 2_048,
}


class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



					'''simple docstring'''






					UpperCAmelCase__       :						Union[str, Any]      =   VOCAB_FILES_NAMES
					UpperCAmelCase__       :						str      =   PRETRAINED_VOCAB_FILES_MAP
					UpperCAmelCase__       :						Any      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCAmelCase__       :						Any      =   ['''input_ids''', '''attention_mask''']

					def __init__(					self						,   lowerCamelCase						,   lowerCamelCase="<s>"						,   lowerCamelCase="</s>"						,   lowerCamelCase="</s>"						,   lowerCamelCase="<s>"						,   lowerCamelCase="<unk>"						,   lowerCamelCase="<pad>"						,   lowerCamelCase = None						,   **lowerCamelCase						,   ):
							_snake_case     		=				{} if sp_model_kwargs is None else sp_model_kwargs

							# Compatibility with the original tokenizer
							_snake_case     		=				7
							_snake_case     		=				[F'''<madeupword{i}>''' for i in range(self.num_madeup_words	)]

							_snake_case     		=				kwargs.get("additional_special_tokens"						,   []	)
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    bos_token=lowerCamelCase						,   eos_token=lowerCamelCase						,   unk_token=lowerCamelCase						,   sep_token=lowerCamelCase						,   cls_token=lowerCamelCase						,   pad_token=lowerCamelCase						,   sp_model_kwargs=self.sp_model_kwargs						,   **lowerCamelCase						,   )

							_snake_case     		=				spm.SentencePieceProcessor(**self.sp_model_kwargs	)
							self.sp_model.Load(str(lowerCamelCase	)	)
							_snake_case     		=				vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_snake_case     		=				1

							# Mimic fairseq token-to-id alignment for the first 4 token
							_snake_case     		=				{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

							_snake_case     		=				len(self.sp_model	)
							_snake_case     		=				{F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words	)}
							self.fairseq_tokens_to_ids.update(lowerCamelCase	)

							_snake_case     		=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}

					def __getstate__(					self	):
							_snake_case     		=				self.__dict__.copy()
							_snake_case     		=				None
							_snake_case     		=				self.sp_model.serialized_model_proto()
							return state

					def __setstate__(					self						,   lowerCamelCase	):
							_snake_case     		=				d

							# for backward compatibility
							if not hasattr(self						,   "sp_model_kwargs"	):
									_snake_case     		=				{}

							_snake_case     		=				spm.SentencePieceProcessor(**self.sp_model_kwargs	)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto	)

					def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):

							if token_ids_a is None:
									return [self.sep_token_id] + token_ids_a
							_snake_case     		=				[self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a

					def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None						,   lowerCamelCase = False	):

							if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCamelCase						,   token_ids_a=lowerCamelCase						,   already_has_special_tokens=lowerCamelCase	)

							if token_ids_a is None:
									return [1] + ([0] * len(lowerCamelCase	))
							return [1] + ([0] * len(lowerCamelCase	)) + [1, 1] + ([0] * len(lowerCamelCase	))

					def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):
							_snake_case     		=				[self.sep_token_id]

							if token_ids_a is None:
									return len(sep + token_ids_a	) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a	) * [0]

					@property
					def 	UpperCamelCase(					self	):
							return len(self.sp_model	) + self.fairseq_offset + self.num_madeup_words

					def 	UpperCamelCase(					self	):
							_snake_case     		=				{self.convert_ids_to_tokens(lowerCamelCase	): i for i in range(self.vocab_size	)}
							vocab.update(self.added_tokens_encoder	)
							return vocab

					def 	UpperCamelCase(					self						,   lowerCamelCase	):
							return self.sp_model.encode(lowerCamelCase						,   out_type=lowerCamelCase	)

					def 	UpperCamelCase(					self						,   lowerCamelCase	):
							if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
							_snake_case     		=				self.sp_model.PieceToId(lowerCamelCase	)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

					def 	UpperCamelCase(					self						,   lowerCamelCase	):
							if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset	)

					def 	UpperCamelCase(					self						,   lowerCamelCase	):
							_snake_case     		=				"".join(lowerCamelCase	).replace(lowerCamelCase						,   " "	).strip()
							return out_string






					def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):
							if not os.path.isdir(lowerCamelCase	):
									logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
									return
							_snake_case     		=				os.path.join(
							    lowerCamelCase						,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

							if os.path.abspath(self.vocab_file	) != os.path.abspath(lowerCamelCase	) and os.path.isfile(self.vocab_file	):
									copyfile(self.vocab_file						,   lowerCamelCase	)
							elif not os.path.isfile(self.vocab_file	):
									with open(lowerCamelCase						,   "wb"	) as fi:
											_snake_case     		=				self.sp_model.serialized_model_proto()
											fi.write(lowerCamelCase	)

							return (out_vocab_file,)









'''simple docstring'''



from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_torch_available
from ...utils import OptionalDependencyNotAvailable


__magic_name__   :						Any    =			{
    """configuration_gpt_neox_japanese""": ["""GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTNeoXJapaneseConfig"""],
    """tokenization_gpt_neox_japanese""": ["""GPTNeoXJapaneseTokenizer"""],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __magic_name__   :						List[Any]    =			[
     """GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """GPTNeoXJapaneseForCausalLM""",
     """GPTNeoXJapaneseLayer""",
     """GPTNeoXJapaneseModel""",
     """GPTNeoXJapanesePreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_gpt_neox_japanese import GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXJapaneseConfig
 from .tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_gpt_neox_japanese import (
      GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
      GPTNeoXJapaneseForCausalLM,
      GPTNeoXJapaneseLayer,
      GPTNeoXJapaneseModel,
      GPTNeoXJapanesePreTrainedModel,
  )


else:
 import sys

 __magic_name__   :						Union[str, Any]    =			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


