'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case						  =		logging.get_logger(__name__)

__snake_case						  =		"""▁"""

__snake_case						  =		{"""vocab_file""": """sentencepiece.bpe.model"""}

__snake_case						  =		{
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

__snake_case						  =		{
    """xlm-roberta-base""": 512,
    """xlm-roberta-large""": 512,
    """xlm-roberta-large-finetuned-conll02-dutch""": 512,
    """xlm-roberta-large-finetuned-conll02-spanish""": 512,
    """xlm-roberta-large-finetuned-conll03-english""": 512,
    """xlm-roberta-large-finetuned-conll03-german""": 512,
}





class       _a    (		__a      ):


  """simple docstring"""

  A_    							=						VOCAB_FILES_NAMES
  A_    							=						PRETRAINED_VOCAB_FILES_MAP
  A_    							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A_    							=						['''input_ids''', '''attention_mask''']




  def __init__(		self      :       Dict   ,				lowercase_      :       Optional[Any]   ,				lowercase_      :       List[str]="<s>"   ,				lowercase_      :       List[str]="</s>"   ,				lowercase_      :       Optional[Any]="</s>"   ,				lowercase_      :       Optional[Any]="<s>"   ,				lowercase_      :       List[Any]="<unk>"   ,				lowercase_      :       List[Any]="<pad>"   ,				lowercase_      :       Dict="<mask>"   ,				lowercase_      :       Optional[Dict[str, Any]] = None   ,				**lowercase_      :       str   ,				):







    '''simple docstring'''


    lowercase_     =       AddedToken(lowercase_   ,				lstrip=lowercase_   ,				rstrip=lowercase_      ) if isinstance(lowercase_   ,				lowercase_      ) else mask_token

    lowercase_     =       {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        bos_token=lowercase_   ,				eos_token=lowercase_   ,				unk_token=lowercase_   ,				sep_token=lowercase_   ,				cls_token=lowercase_   ,				pad_token=lowercase_   ,				mask_token=lowercase_   ,				sp_model_kwargs=self.sp_model_kwargs   ,				**lowercase_   ,				)

    lowercase_     =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
    self.sp_model.Load(str(lowercase_      )      )
    lowercase_     =       vocab_file

    # Original fairseq vocab and spm vocab must be "aligned":
    # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
    # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
    # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
    # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

    # Mimic fairseq token-to-id alignment for the first 4 token
    lowercase_     =       {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

    # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
    lowercase_     =       1

    lowercase_     =       len(self.sp_model      ) + self.fairseq_offset
    lowercase_     =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




  def __getstate__(		self      :       Tuple      ):







    '''simple docstring'''


    lowercase_     =       self.__dict__.copy()
    lowercase_     =       None
    lowercase_     =       self.sp_model.serialized_model_proto()
    return state




  def __setstate__(		self      :       Any   ,				lowercase_      :       Dict      ):







    '''simple docstring'''


    lowercase_     =       d

    # for backward compatibility
    if not hasattr(self   ,				"""sp_model_kwargs"""      ):
      lowercase_     =       {}

    lowercase_     =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
    self.sp_model.LoadFromSerializedProto(self.sp_model_proto      )




  def 		lowerCamelCase__  (		self      :       List[str]   ,				lowercase_      :       List[int]   ,				lowercase_      :       Optional[List[int]] = None      ):







    '''simple docstring'''


    if token_ids_a is None:
      return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    lowercase_     =       [self.cls_token_id]
    lowercase_     =       [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep




  def 		lowerCamelCase__  (		self      :       int   ,				lowercase_      :       List[int]   ,				lowercase_      :       Optional[List[int]] = None   ,				lowercase_      :       bool = False      ):







    '''simple docstring'''


    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=lowercase_   ,				token_ids_a=lowercase_   ,				already_has_special_tokens=lowercase_      )

    if token_ids_a is None:
      return [1] + ([0] * len(lowercase_      )) + [1]
    return [1] + ([0] * len(lowercase_      )) + [1, 1] + ([0] * len(lowercase_      )) + [1]




  def 		lowerCamelCase__  (		self      :       List[Any]   ,				lowercase_      :       List[int]   ,				lowercase_      :       Optional[List[int]] = None      ):







    '''simple docstring'''


    lowercase_     =       [self.sep_token_id]
    lowercase_     =       [self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep      ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]




  @property
  def 		lowerCamelCase__  (		self      :       Optional[Any]      ):







    '''simple docstring'''


    return len(self.sp_model      ) + self.fairseq_offset + 1  # Add the <mask> token




  def 		lowerCamelCase__  (		self      :       List[str]      ):







    '''simple docstring'''


    lowercase_     =       {self.convert_ids_to_tokens(lowercase_      ): i for i in range(self.vocab_size      )}
    vocab.update(self.added_tokens_encoder      )
    return vocab




  def 		lowerCamelCase__  (		self      :       Optional[int]   ,				lowercase_      :       str      ):







    '''simple docstring'''


    return self.sp_model.encode(lowercase_   ,				out_type=lowercase_      )




  def 		lowerCamelCase__  (		self      :       Dict   ,				lowercase_      :       Tuple      ):







    '''simple docstring'''


    if token in self.fairseq_tokens_to_ids:
      return self.fairseq_tokens_to_ids[token]
    lowercase_     =       self.sp_model.PieceToId(lowercase_      )

    # Need to return unknown token if the SP model returned 0
    return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




  def 		lowerCamelCase__  (		self      :       Union[str, Any]   ,				lowercase_      :       int      ):







    '''simple docstring'''


    if index in self.fairseq_ids_to_tokens:
      return self.fairseq_ids_to_tokens[index]
    return self.sp_model.IdToPiece(index - self.fairseq_offset      )




  def 		lowerCamelCase__  (		self      :       List[Any]   ,				lowercase_      :       List[str]      ):







    '''simple docstring'''


    lowercase_     =       """""".join(lowercase_      ).replace(lowercase_   ,				""" """      ).strip()
    return out_string





  def 		lowerCamelCase__  (		self      :       Optional[int]   ,				lowercase_      :       str   ,				lowercase_      :       Optional[str] = None      ):







    '''simple docstring'''


    if not os.path.isdir(lowercase_      ):
      logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
      return
    lowercase_     =       os.path.join(
        lowercase_   ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

    if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowercase_      ) and os.path.isfile(self.vocab_file      ):
      copyfile(self.vocab_file   ,				lowercase_      )
    elif not os.path.isfile(self.vocab_file      ):
      with open(lowercase_   ,				"""wb"""      ) as fi:
        lowercase_     =       self.sp_model.serialized_model_proto()
        fi.write(lowercase_      )

    return (out_vocab_file,)


'''simple docstring'''


import math

def   A_      (       SCREAMING_SNAKE_CASE_						)			->int:

  if not isinstance(SCREAMING_SNAKE_CASE_ ,	SCREAMING_SNAKE_CASE_						):
    lowercase_     =       f"""Input value of [number={number}] must be an integer"""
    raise TypeError(SCREAMING_SNAKE_CASE_						)

  if number < 1:
    lowercase_     =       f"""Input value of [number={number}] must be > 0"""
    raise ValueError(SCREAMING_SNAKE_CASE_						)
  elif number == 1:
    return 3
  elif number == 2:
    return 5
  else:
    lowercase_     =       int(math.log(number // 3 ,	2						)						) + 2

    lowercase_     =       [3, 5]
    lowercase_     =       2
    lowercase_     =       3
    for block in range(1 ,	SCREAMING_SNAKE_CASE_						):
      for _ in range(SCREAMING_SNAKE_CASE_						):
        proth_list.append(2 ** (block + 1) + proth_list[proth_index - 1]						)
        proth_index += 1
      increment *= 2

  return proth_list[number - 1]


if __name__ == "__main__":
       import doctest

       doctest.testmod()

       for number in range(11):
              __snake_case						  =		0
              try:
                     __snake_case						  =		proth(number)
              except ValueError:
                     print(f'''ValueError: there is no {number}th Proth number''')
                     continue

              print(f'''The {number}th Proth number: {value}''')

