


import logging
import os
import threading
import time


try:
       import warnings
except ImportError:
       __UpperCamelCase       :       Any						       =				None

try:
       import msvcrt
except ImportError:
       __UpperCamelCase       :       Optional[Any]						       =				None

try:
       import fcntl
except ImportError:
       __UpperCamelCase       :       Union[str, Any]						       =				None


# Backward compatibility
# ------------------------------------------------
try:
       TimeoutError
except NameError:
       __UpperCamelCase       :       Any						       =				OSError


# Data
# ------------------------------------------------
__UpperCamelCase       :       Optional[int]						       =				[
    "Timeout",
    "BaseFileLock",
    "WindowsFileLock",
    "UnixFileLock",
    "SoftFileLock",
    "FileLock",
]

__UpperCamelCase       :       Dict						       =				"3.0.12"


__UpperCamelCase       :       str						       =				None

def 			_a  (				):







 """simple docstring"""



 global _logger
 UpperCamelCase__					:    Tuple          =   _logger or logging.getLogger(__name__     )
 return _logger

class       __magic_name__	(     __lowerCAmelCase):


     def __init__(      self							:     Any							,	lowerCamelCase__							:     List[Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   lock_file
      return None


     def __str__(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Tuple          =   F"The file lock '{self.lock_file}' could not be acquired."
      return temp



class       __magic_name__	:


     def __init__(      self							:     List[str]							,	lowerCamelCase__							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   lock
      return None


     def __enter__(      self							:     Tuple			)  ->      str:





      '''simple docstring'''






      return self.lock


     def __exit__(      self							:     Any							,	lowerCamelCase__							:     List[Any]							,	lowerCamelCase__							:     Dict							,	lowerCamelCase__							:     str			)  ->      Any:





      '''simple docstring'''






      self.lock.release()
      return None



class       __magic_name__	:


     def __init__(      self							:     List[Any]							,	lowerCamelCase__							:     int							,	lowerCamelCase__							:     List[str]=-1							,	lowerCamelCase__							:     Any=None			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   max_filename_length if max_filename_length is not None else 255
      # Hash the filename if it's too long
      UpperCamelCase__					:    List[Any]          =   self.hash_filename_if_too_long(lowerCamelCase__							,	lowerCamelCase__			)
      # The path to the lock file.
      UpperCamelCase__					:    Tuple          =   lock_file

      # The file descriptor for the *_lock_file* as it is returned by the
      # os.open() function.
      # This file lock is only NOT None, if the object currently holds the
      # lock.
      UpperCamelCase__					:    int          =   None

      # The default timeout value.
      UpperCamelCase__					:    Optional[Any]          =   timeout

      # We use this lock primarily for the lock counter.
      UpperCamelCase__					:    List[Any]          =   threading.Lock()

      # The lock counter is used for implementing the nested locking
      # mechanism. Whenever the lock is acquired, the counter is increased and
      # the lock is only released, when this value is 0 again.
      UpperCamelCase__					:    Any          =   0
      return None


     @property
     def      UpperCAmelCase__	(      self							:     Any			)  ->      Optional[int]:





      '''simple docstring'''






      return self._lock_file


     @property
     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Optional[int]:





      '''simple docstring'''






      return self._timeout


     @timeout.setter
     def      UpperCAmelCase__	(      self							:     List[Any]							,	lowerCamelCase__							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   float(lowerCamelCase__			)
      return None


     def      UpperCAmelCase__	(      self							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      raise NotImplementedError()


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Dict:





      '''simple docstring'''






      raise NotImplementedError()


     @property
     def      UpperCAmelCase__	(      self							:     Any			)  ->      Dict:





      '''simple docstring'''






      return self._lock_file_fd is not None


     def      UpperCAmelCase__	(      self							:     Tuple							,	lowerCamelCase__							:     Optional[Any]=None							,	lowerCamelCase__							:     List[str]=0.05			)  ->      int:





      '''simple docstring'''






      if timeout is None:
       UpperCamelCase__					:    int          =   self.timeout

      # Increment the number right at the beginning.
      # We can still undo it, if something fails.
      with self._thread_lock:
       self._lock_counter += 1

      UpperCamelCase__					:    int          =   id(self			)
      UpperCamelCase__					:    List[Any]          =   self._lock_file
      UpperCamelCase__					:    Dict          =   time.time()
      try:
       while True:
        with self._thread_lock:
         if not self.is_locked:
          logger().debug(F"Attempting to acquire lock {lock_id} on {lock_filename}"			)
          self._acquire()

        if self.is_locked:
         logger().debug(F"Lock {lock_id} acquired on {lock_filename}"			)
         break
        elif timeout >= 0 and time.time() - start_time > timeout:
         logger().debug(F"Timeout on acquiring lock {lock_id} on {lock_filename}"			)
         raise Timeout(self._lock_file			)
        else:
         logger().debug(
             F"Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ..."			)
         time.sleep(lowerCamelCase__			)
      except:  # noqa
       # Something did go wrong, so decrement the counter.
       with self._thread_lock:
        UpperCamelCase__					:    List[Any]          =   max(0							,	self._lock_counter - 1			)

       raise
      return _Acquire_ReturnProxy(lock=self			)


     def      UpperCAmelCase__	(      self							:     Optional[int]							,	lowerCamelCase__							:     Optional[int]=False			)  ->      Any:





      '''simple docstring'''






      with self._thread_lock:
       if self.is_locked:
        self._lock_counter -= 1

        if self._lock_counter == 0 or force:
         UpperCamelCase__					:    List[str]          =   id(self			)
         UpperCamelCase__					:    Union[str, Any]          =   self._lock_file

         logger().debug(F"Attempting to release lock {lock_id} on {lock_filename}"			)
         self._release()
         UpperCamelCase__					:    Optional[Any]          =   0
         logger().debug(F"Lock {lock_id} released on {lock_filename}"			)

      return None


     def __enter__(      self							:     Union[str, Any]			)  ->      Tuple:





      '''simple docstring'''






      self.acquire()
      return self


     def __exit__(      self							:     Optional[int]							,	lowerCamelCase__							:     List[str]							,	lowerCamelCase__							:     Any							,	lowerCamelCase__							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      self.release()
      return None


     def __del__(      self							:     Optional[Any]			)  ->      List[str]:





      '''simple docstring'''






      self.release(force=lowerCamelCase__			)
      return None


     def      UpperCAmelCase__	(      self							:     Optional[int]							,	lowerCamelCase__							:     str							,	lowerCamelCase__							:     int			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   os.path.basename(lowerCamelCase__			)
      if len(lowerCamelCase__			) > max_length and max_length > 0:
       UpperCamelCase__					:    Optional[int]          =   os.path.dirname(lowerCamelCase__			)
       UpperCamelCase__					:    List[str]          =   str(hash(lowerCamelCase__			)			)
       UpperCamelCase__					:    Optional[int]          =   filename[: max_length - len(lowerCamelCase__			) - 8] + '''...''' + hashed_filename + '''.lock'''
       return os.path.join(lowerCamelCase__							,	lowerCamelCase__			)
      else:
       return path



class       __magic_name__	(     __lowerCAmelCase):


     def __init__(      self							:     int							,	lowerCamelCase__							:     Dict							,	lowerCamelCase__							:     str=-1							,	lowerCamelCase__							:     int=None			)  ->      Optional[int]:





      '''simple docstring'''






      from .file_utils import relative_to_absolute_path

      super().__init__(lowerCamelCase__							,	timeout=lowerCamelCase__							,	max_filename_length=lowerCamelCase__			)
      UpperCamelCase__					:    int          =   '''\\\\?\\''' + relative_to_absolute_path(self.lock_file			)


     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   os.O_RDWR | os.O_CREAT | os.O_TRUNC

      try:
       UpperCamelCase__					:    Union[str, Any]          =   os.open(self._lock_file							,	lowerCamelCase__			)
      except OSError:
       pass
      else:
       try:
        msvcrt.locking(lowerCamelCase__							,	msvcrt.LK_NBLCK							,	1			)
       except OSError:
        os.close(lowerCamelCase__			)
       else:
        UpperCamelCase__					:    Union[str, Any]          =   fd
      return None


     def      UpperCAmelCase__	(      self							:     Tuple			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   self._lock_file_fd
      UpperCamelCase__					:    List[str]          =   None
      msvcrt.locking(lowerCamelCase__							,	msvcrt.LK_UNLCK							,	1			)
      os.close(lowerCamelCase__			)

      try:
       os.remove(self._lock_file			)
      # Probably another instance of the application
      # that acquired the file lock.
      except OSError:
       pass
      return None



class       __magic_name__	(     __lowerCAmelCase):


     def __init__(      self							:     Any							,	lowerCamelCase__							:     int							,	lowerCamelCase__							:     str=-1							,	lowerCamelCase__							:     int=None			)  ->      Tuple:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   os.statvfs(os.path.dirname(lowerCamelCase__			)			).f_namemax
      super().__init__(lowerCamelCase__							,	timeout=lowerCamelCase__							,	max_filename_length=lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   os.O_RDWR | os.O_CREAT | os.O_TRUNC
      UpperCamelCase__					:    int          =   os.open(self._lock_file							,	lowerCamelCase__			)

      try:
       fcntl.flock(lowerCamelCase__							,	fcntl.LOCK_EX | fcntl.LOCK_NB			)
      except OSError:
       os.close(lowerCamelCase__			)
      else:
       UpperCamelCase__					:    Any          =   fd
      return None


     def      UpperCAmelCase__	(      self							:     int			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Tuple          =   self._lock_file_fd
      UpperCamelCase__					:    int          =   None
      fcntl.flock(lowerCamelCase__							,	fcntl.LOCK_UN			)
      os.close(lowerCamelCase__			)
      return None



class       __magic_name__	(     __lowerCAmelCase):


     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      List[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
      try:
       UpperCamelCase__					:    Any          =   os.open(self._lock_file							,	lowerCamelCase__			)
      except OSError:
       pass
      else:
       UpperCamelCase__					:    Optional[Any]          =   fd
      return None


     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      Optional[Any]:





      '''simple docstring'''






      os.close(self._lock_file_fd			)
      UpperCamelCase__					:    List[str]          =   None

      try:
       os.remove(self._lock_file			)
      # The file is already deleted and that's what we want.
      except OSError:
       pass
      return None



__UpperCamelCase       :       Tuple						       =				None

if msvcrt:
       __UpperCamelCase       :       str						       =				WindowsFileLock
elif fcntl:
       __UpperCamelCase       :       Optional[Any]						       =				UnixFileLock
else:
       __UpperCamelCase       :       Optional[Any]						       =				SoftFileLock

       if warnings is not None:
              warnings.warn("only soft file lock is available")






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   mock.Mock()
      UpperCamelCase__					:    Tuple          =   500
      UpperCamelCase__					:    Dict          =   {}
      UpperCamelCase__					:    Dict          =   HTTPError
      UpperCamelCase__					:    Optional[Any]          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    List[Any]          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    Any          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     str			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   mock.Mock()
      UpperCamelCase__					:    Any          =   500
      UpperCamelCase__					:    str          =   {}
      UpperCamelCase__					:    Optional[Any]          =   HTTPError
      UpperCamelCase__					:    Tuple          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    Optional[Any]          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    str          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      try:
       UpperCamelCase__					:    Dict          =   tempfile.mktemp()
       with open(lowerCamelCase__							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							,	lowerCamelCase__			)

       UpperCamelCase__					:    Dict          =   AlbertTokenizer.from_pretrained(lowerCamelCase__			)
      finally:
       os.remove(lowerCamelCase__			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''			):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json'''							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							,	lowerCamelCase__			)
       UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''			)
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size							,	1000			)
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			)



@is_staging_test
class       __magic_name__	(     unittest.TestCase):
     A:		List[Any]			=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


     @classmethod
     def      UpperCAmelCase__	(      cls							:     List[str]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   TOKEN
      HfFolder.save_token(lowerCamelCase__			)


     @classmethod
     def      UpperCAmelCase__	(      cls							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      try:
       delete_repo(token=cls._token							,	repo_id='''test-tokenizer'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''valid_org/test-tokenizer-org'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''test-dynamic-tokenizer'''			)
      except HTTPError:
       pass


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    int          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Tuple          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-tokenizer'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''test-tokenizer'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(lowerCamelCase__							,	repo_id='''test-tokenizer'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    Dict          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    List[str]          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Optional[Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''valid_org/test-tokenizer-org'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           lowerCamelCase__							,	repo_id='''valid_org/test-tokenizer-org'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      int:





      '''simple docstring'''






      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Any          =   CustomTokenizer(lowerCamelCase__			)

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)

       UpperCamelCase__					:    List[Any]          =   BertTokenizerFast.from_pretrained(lowerCamelCase__			)
       bert_tokenizer.save_pretrained(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[int]          =   CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizerFast'''			)
      UpperCamelCase__					:    int          =   AutoTokenizer.from_pretrained(
          F"{USER}/test-dynamic-tokenizer"							,	use_fast=lowerCamelCase__							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)



class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     str			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''Hello 友達'''			)
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)
      trie.add('''Hello'''			)
      trie.data
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS] This is a extra_id_100''']			)
      trie.add('''[CLS]'''			)
      trie.add('''extra_id_1'''			)
      trie.add('''extra_id_100'''			)
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS]''', ''' This is a ''', '''extra_id_100''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''A'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''A''', '''BC''']			)
      self.assertEqual(trie.split('''BCA'''			)							,	['''BC''', '''A''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''TOKEN]'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''A'''			)
      trie.add('''P'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''AB'''			)
      trie.add('''B'''			)
      trie.add('''C'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''AB''', '''C''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''ABC'''			)
      trie.add('''B'''			)
      trie.add('''CD'''			)
      self.assertEqual(trie.split('''ABCD'''			)							,	['''ABC''', '''D''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      UpperCamelCase__					:    Optional[Any]          =   trie.cut_text('''ABC'''							,	[0, 0, 2, 1, 2, 3]			)
      self.assertEqual(lowerCamelCase__							,	['''AB''', '''C''']			)


