
"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a								=						logging.get_logger(__name__)

_a								=						"""▁"""

_a								=						{"""vocab_file""": """sentencepiece.bpe.model"""}

_a								=						{
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    }
}

_a								=						{
    """facebook/xglm-564M""": 2048,
}






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = VOCAB_FILES_NAMES
		lowercase__         = PRETRAINED_VOCAB_FILES_MAP
		lowercase__         = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase__         = ['input_ids', 'attention_mask']



		def __init__(   self							,						__a							,						__a="<s>"							,						__a="</s>"							,						__a="</s>"							,						__a="<s>"							,						__a="<unk>"							,						__a="<pad>"							,						__a = None							,						**__a							,						)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    {} if sp_model_kwargs is None else sp_model_kwargs

							# Compatibility with the original tokenizer
							_UpperCamelCase									=    7
							_UpperCamelCase									=    [F'''<madeupword{i}>''' for i in range(self.num_madeup_words)]

							_UpperCamelCase									=    kwargs.get('''additional_special_tokens'''							,						[])
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    bos_token=__a							,						eos_token=__a							,						unk_token=__a							,						sep_token=__a							,						cls_token=__a							,						pad_token=__a							,						sp_model_kwargs=self.sp_model_kwargs							,						**__a							,						)

							_UpperCamelCase									=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(str(__a))
							_UpperCamelCase									=    vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_UpperCamelCase									=    1

							# Mimic fairseq token-to-id alignment for the first 4 token
							_UpperCamelCase									=    {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							_UpperCamelCase									=    len(self.sp_model)
							_UpperCamelCase									=    {F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words)}
							self.fairseq_tokens_to_ids.update(__a)

							_UpperCamelCase									=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}



		def __getstate__(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.__dict__.copy()
							_UpperCamelCase									=    None
							_UpperCamelCase									=    self.sp_model.serialized_model_proto()
							return state



		def __setstate__(   self							,						__a)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    d

							# for backward compatibility
							if not hasattr(self							,						'''sp_model_kwargs'''):
												_UpperCamelCase									=    {}

							_UpperCamelCase									=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto)



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[int]:






							'''simple docstring'''

							if token_ids_a is None:
												return [self.sep_token_id] + token_ids_a
							_UpperCamelCase									=    [self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = False)			->			List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a							,						token_ids_a=__a							,						already_has_special_tokens=__a)

							if token_ids_a is None:
												return [1] + ([0] * len(__a))
							return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a))



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[int]:






							'''simple docstring'''
							_UpperCamelCase									=    [self.sep_token_id]

							if token_ids_a is None:
												return len(sep + token_ids_a) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a) * [0]



		@property
		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return len(self.sp_model) + self.fairseq_offset + self.num_madeup_words



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							_UpperCamelCase									=    {self.convert_ids_to_tokens(__a): i for i in range(self.vocab_size)}
							vocab.update(self.added_tokens_encoder)
							return vocab



		def 		UpperCAmelCase				(   self							,						__a)			->			List[str]:






							'''simple docstring'''
							return self.sp_model.encode(__a							,						out_type=__a)



		def 		UpperCAmelCase				(   self							,						__a)			->			Union[str, Any]:






							'''simple docstring'''
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							_UpperCamelCase									=    self.sp_model.PieceToId(__a)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def 		UpperCAmelCase				(   self							,						__a)			->			Tuple:






							'''simple docstring'''
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset)



		def 		UpperCAmelCase				(   self							,						__a)			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    ''''''.join(__a).replace(__a							,						''' ''').strip()
							return out_string




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			Tuple[str]:






							'''simple docstring'''
							if not os.path.isdir(__a):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
												return
							_UpperCamelCase									=    os.path.join(
							    __a							,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__a) and os.path.isfile(self.vocab_file):
												copyfile(self.vocab_file							,						__a)
							elif not os.path.isfile(self.vocab_file):
												with open(__a							,						'''wb''') as fi:
																	_UpperCamelCase									=    self.sp_model.serialized_model_proto()
																	fi.write(__a)

							return (out_vocab_file,)











"""simple docstring"""






from copy import deepcopy



class     UpperCAmelCase    :



   def __init__(       self    :	Optional[Any]							,       __lowerCamelCase    :	list[int] | None = None							,       __lowerCamelCase    :	int | None = None				):



     """simple docstring"""

     if arr is None and size is not None:
       _snake_case   	=      size
       _snake_case   	=      [0] * size
     elif arr is not None:
       self.init(__lowerCamelCase				)
     else:
       raise ValueError('''Either arr or size must be specified'''				)



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	list[int]				):



     """simple docstring"""
     _snake_case   	=      len(__lowerCamelCase				)
     _snake_case   	=      deepcopy(__lowerCamelCase				)
     for i in range(1							,       self.size				):
       _snake_case   	=      self.next_(__lowerCamelCase				)
       if j < self.size:
         self.tree[j] += self.tree[i]



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      self.tree[:]
     for i in range(self.size - 1							,       0							,       -1				):
       _snake_case   	=      self.next_(__lowerCamelCase				)
       if j < self.size:
         arr[j] -= arr[i]
     return arr



   @staticmethod
   def    __UpperCAmelCase    (       __lowerCamelCase    :	int				):



     """simple docstring"""
     return index + (index & (-index))



   @staticmethod
   def    __UpperCAmelCase    (       __lowerCamelCase    :	int				):



     """simple docstring"""
     return index - (index & (-index))



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if index == 0:
       self.tree[0] += value
       return
     while index < self.size:
       self.tree[index] += value
       _snake_case   	=      self.next_(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     self.add(__lowerCamelCase							,       value - self.get(__lowerCamelCase				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if right == 0:
       return 0
     _snake_case   	=      self.tree[0]
     right -= 1  # make right inclusive
     while right > 0:
       result += self.tree[right]
       _snake_case   	=      self.prev(__lowerCamelCase				)
     return result



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.prefix(__lowerCamelCase				) - self.prefix(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.query(__lowerCamelCase							,       index + 1				)





   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     value -= self.tree[0]
     if value < 0:
       return -1

     _snake_case   	=      1  # Largest power of 2 <= size
     while j * 2 < self.size:
       j *= 2

     _snake_case   	=      0

     while j > 0:
       if i + j < self.size and self.tree[i + j] <= value:
         value -= self.tree[i + j]
         i += j
       j //= 2
     return i


if __name__ == "__main__":
     import doctest

     doctest.testmod()





