


'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =			'▁'

SCREAMING_SNAKE_CASE_      =			{'vocab_file': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE_      =			{
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

SCREAMING_SNAKE_CASE_      =			{
    'facebook/xglm-564M': 20_48,
}


class  lowerCAmelCase_						(       snake_case__      ):





  """simple docstring"""



  a_       :Optional[Any]												=VOCAB_FILES_NAMES
  a_       :Union[str, Any]												=PRETRAINED_VOCAB_FILES_MAP
  a_       :List[str]												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_       :Optional[int]												=["""input_ids""", """attention_mask"""]

  def __init__(					self     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="<s>"				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]="</s>"				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]="</s>"				,							SCREAMING_SNAKE_CASE__     :							str="<s>"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="<unk>"				,							SCREAMING_SNAKE_CASE__     :							Tuple="<pad>"				,							SCREAMING_SNAKE_CASE__     :							Optional[Dict[str, Any]] = None				,							**SCREAMING_SNAKE_CASE__     :							Optional[int]				,							):





       '''simple docstring'''

       __a       =   {} if sp_model_kwargs is None else sp_model_kwargs

       # Compatibility with the original tokenizer
       __a       =   7
       __a       =   [f'''<madeupword{i}>''' for i in range(self.num_madeup_words						)]

       __a       =   kwargs.get("""additional_special_tokens"""				,							[]						)
       kwargs["additional_special_tokens"] += [
           word for word in madeup_words if word not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           bos_token=SCREAMING_SNAKE_CASE__				,							eos_token=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							sp_model_kwargs=self.sp_model_kwargs				,							**SCREAMING_SNAKE_CASE__				,							)

       __a       =   spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(str(SCREAMING_SNAKE_CASE__						)						)
       __a       =   vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       __a       =   1

       # Mimic fairseq token-to-id alignment for the first 4 token
       __a       =   {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

       __a       =   len(self.sp_model						)
       __a       =   {f'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words						)}
       self.fairseq_tokens_to_ids.update(SCREAMING_SNAKE_CASE__						)

       __a       =   {v: k for k, v in self.fairseq_tokens_to_ids.items()}

  def __getstate__(					self     :							Any						):





       '''simple docstring'''

       __a       =   self.__dict__.copy()
       __a       =   None
       __a       =   self.sp_model.serialized_model_proto()
       return state

  def __setstate__(					self     :							str				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       __a       =   d

       # for backward compatibility
       if not hasattr(self				,							"""sp_model_kwargs"""						):
            __a       =   {}

       __a       =   spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)

  def __a						(					self     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       if token_ids_a is None:
            return [self.sep_token_id] + token_ids_a
       __a       =   [self.sep_token_id]
       return sep + token_ids_a + sep + sep + token_ids_a

  def __a						(					self     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None				,							SCREAMING_SNAKE_CASE__     :							bool = False						):





       '''simple docstring'''

       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE__				,							token_ids_a=SCREAMING_SNAKE_CASE__				,							already_has_special_tokens=SCREAMING_SNAKE_CASE__						)

       if token_ids_a is None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__						))
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__						))

  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]

       if token_ids_a is None:
            return len(sep + token_ids_a						) * [0]
       return len(sep + token_ids_a + sep + sep + token_ids_a						) * [0]

  @property
  def __a						(					self     :							List[Any]						):





       '''simple docstring'''

       return len(self.sp_model						) + self.fairseq_offset + self.num_madeup_words

  def __a						(					self     :							Any						):





       '''simple docstring'''

       __a       =   {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       return self.sp_model.encode(SCREAMING_SNAKE_CASE__				,							out_type=SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							str				,							SCREAMING_SNAKE_CASE__     :							List[Any]						):





       '''simple docstring'''

       if token in self.fairseq_tokens_to_ids:
            return self.fairseq_tokens_to_ids[token]
       __a       =   self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__						)

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

  def __a						(					self     :							Dict				,							SCREAMING_SNAKE_CASE__     :							List[Any]						):





       '''simple docstring'''

       if index in self.fairseq_ids_to_tokens:
            return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset						)

  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							Dict						):





       '''simple docstring'''

       __a       =   """""".join(SCREAMING_SNAKE_CASE__						).replace(SCREAMING_SNAKE_CASE__				,							""" """						).strip()
       return out_string


  def __a						(					self     :							str				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str] = None						):





       '''simple docstring'''

       if not os.path.isdir(SCREAMING_SNAKE_CASE__						):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
            return
       __a       =   os.path.join(
           SCREAMING_SNAKE_CASE__				,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE__						) and os.path.isfile(self.vocab_file						):
            copyfile(self.vocab_file				,							SCREAMING_SNAKE_CASE__						)
       elif not os.path.isfile(self.vocab_file						):
            with open(SCREAMING_SNAKE_CASE__				,							"""wb"""						) as fi:
                 __a       =   self.sp_model.serialized_model_proto()
                 fi.write(SCREAMING_SNAKE_CASE__						)

       return (out_vocab_file,)










'''simple docstring'''







def        __lowercase						(      __SCREAMING_SNAKE_CASE		)				->					str:
     """simple docstring"""




     return " ".join(input_str.split()[::-1]		)


if __name__ == "__main__":
 import doctest

 doctest.testmod()






