


import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE     :					str      =       logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE     :					Any      =       {
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

_SCREAMING_SNAKE_CASE     :					Optional[int]      =       {
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

_SCREAMING_SNAKE_CASE     :					Dict      =       {
    "vinai/phobert-base": 2_56,
    "vinai/phobert-large": 2_56,
}
def      UpperCAmelCase__		(UpperCamelCase_							):



     """simple docstring"""






     snake_case											=							set()
     snake_case											=							word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          snake_case											=							char

     snake_case											=							set(UpperCamelCase_							)
     return pairs

class     A__   (       snake_case__      ):







    """simple docstring"""

    __magic_name__         =						VOCAB_FILES_NAMES
    __magic_name__         =						PRETRAINED_VOCAB_FILES_MAP
    __magic_name__         =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    def __init__(			self     ,   __snake_case     ,   __snake_case     ,   __snake_case="<s>"     ,   __snake_case="</s>"     ,   __snake_case="</s>"     ,   __snake_case="<s>"     ,   __snake_case="<unk>"     ,   __snake_case="<pad>"     ,   __snake_case="<mask>"     ,   **__snake_case     ,   ):
         super().__init__(
             bos_token=__snake_case     ,   eos_token=__snake_case     ,   unk_token=__snake_case     ,   sep_token=__snake_case     ,   cls_token=__snake_case     ,   pad_token=__snake_case     ,   mask_token=__snake_case     ,   **__snake_case     ,   )

         snake_case											=							vocab_file
         snake_case											=							merges_file

         snake_case											=							{}
         snake_case											=							0
         snake_case											=							1
         snake_case											=							2
         snake_case											=							3

         self.add_from_file(__snake_case       )

         snake_case											=							{v: k for k, v in self.encoder.items()}

         with open(__snake_case     ,   encoding='''utf-8'''       ) as merges_handle:
              snake_case											=							merges_handle.read().split('''\n'''       )[:-1]
         snake_case											=							[tuple(merge.split()[:-1]       ) for merge in merges]
         snake_case											=							dict(zip(__snake_case     ,   range(len(__snake_case       )       )       )       )
         snake_case											=							{}
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None       ):

         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         snake_case											=							[self.cls_token_id]
         snake_case											=							[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None     ,   __snake_case = False       ):

         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=__snake_case     ,   token_ids_a=__snake_case     ,   already_has_special_tokens=__snake_case       )

         if token_ids_a is None:
              return [1] + ([0] * len(__snake_case       )) + [1]
         return [1] + ([0] * len(__snake_case       )) + [1, 1] + ([0] * len(__snake_case       )) + [1]
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None       ):
         snake_case											=							[self.sep_token_id]
         snake_case											=							[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
    @property
    def 						a_     (			self       ):
         return len(self.encoder       )
    def 						a_     (			self       ):
         return dict(self.encoder     ,   **self.added_tokens_encoder       )
    def 						a_     (			self     ,   __snake_case       ):
         if token in self.cache:
              return self.cache[token]
         snake_case											=							tuple(__snake_case       )
         snake_case											=							tuple(list(word[:-1]       ) + [word[-1] + '''</w>''']       )
         snake_case											=							get_pairs(__snake_case       )

         if not pairs:
              return token

         while True:
              snake_case											=							min(__snake_case     ,   key=lambda __snake_case       : self.bpe_ranks.get(__snake_case     ,   float('''inf'''       )       )       )
              if bigram not in self.bpe_ranks:
                   break
              snake_case					,    snake_case											=							bigram
              snake_case											=							[]
              snake_case											=							0
              while i < len(__snake_case       ):
                   try:
                        snake_case											=							word.index(__snake_case     ,   __snake_case       )
                   except ValueError:
                        new_word.extend(word[i:]       )
                        break
                   else:
                        new_word.extend(word[i:j]       )
                        snake_case											=							j

                   if word[i] == first and i < len(__snake_case       ) - 1 and word[i + 1] == second:
                        new_word.append(first + second       )
                        i += 2
                   else:
                        new_word.append(word[i]       )
                        i += 1
              snake_case											=							tuple(__snake_case       )
              snake_case											=							new_word
              if len(__snake_case       ) == 1:
                   break
              else:
                   snake_case											=							get_pairs(__snake_case       )
         snake_case											=							'''@@ '''.join(__snake_case       )
         snake_case											=							word[:-4]
         snake_case											=							word
         return word
    def 						a_     (			self     ,   __snake_case       ):
         snake_case											=							[]

         snake_case											=							re.findall(R'''\S+\n?'''     ,   __snake_case       )

         for token in words:
              split_tokens.extend(list(self.bpe(__snake_case       ).split(''' '''       )       )       )
         return split_tokens
    def 						a_     (			self     ,   __snake_case       ):
         return self.encoder.get(__snake_case     ,   self.encoder.get(self.unk_token       )       )
    def 						a_     (			self     ,   __snake_case       ):
         return self.decoder.get(__snake_case     ,   self.unk_token       )
    def 						a_     (			self     ,   __snake_case       ):
         snake_case											=							''' '''.join(__snake_case       ).replace('''@@ '''     ,   ''''''       ).strip()
         return out_string
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None       ):
         if not os.path.isdir(__snake_case       ):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
              return
         snake_case											=							os.path.join(
             __snake_case     ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
         snake_case											=							os.path.join(
             __snake_case     ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

         if os.path.abspath(self.vocab_file       ) != os.path.abspath(__snake_case       ):
              copyfile(self.vocab_file     ,   __snake_case       )

         if os.path.abspath(self.merges_file       ) != os.path.abspath(__snake_case       ):
              copyfile(self.merges_file     ,   __snake_case       )

         return out_vocab_file, out_merge_file




    def 						a_     (			self     ,   __snake_case       ):
         if isinstance(__snake_case     ,   __snake_case       ):
              try:
                   with open(__snake_case     ,   '''r'''     ,   encoding='''utf-8'''       ) as fd:
                        self.add_from_file(__snake_case       )
              except FileNotFoundError as fnfe:
                   raise fnfe
              except UnicodeError:
                   raise Exception(F'''Incorrect encoding detected in {f}, please rebuild the dataset'''       )
              return

         snake_case											=							f.readlines()
         for lineTmp in lines:
              snake_case											=							lineTmp.strip()
              snake_case											=							line.rfind(''' '''       )
              if idx == -1:
                   raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''       )
              snake_case											=							line[:idx]
              snake_case											=							len(self.encoder       )








from ..utils import DummyObject, requires_backends

class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )






class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )






class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )






class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




