import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast






class 		_lowerCamelCase		(   unittest.TestCase    ):
    def 						UpperCamelCase_				(  self   )       ->					str:
     # A mock response for an HTTP head request to emulate server down
     SCREAMING_SNAKE_CASE__:   Any=      mock.Mock()
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      500
     SCREAMING_SNAKE_CASE__:   Optional[int]=      {}
     SCREAMING_SNAKE_CASE__:   Tuple=      HTTPError
     SCREAMING_SNAKE_CASE__:   Any=      {}

     # Download this model to make sure it's in the cache.
     SCREAMING_SNAKE_CASE__:   Optional[int]=      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )

     # Under the mock environment we get a 500 error when trying to reach the tokenizer.
     with mock.patch('''requests.Session.request''' , return_value=lowerCAmelCase   ) as mock_head:
      SCREAMING_SNAKE_CASE__:   Tuple=      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )
      # This check we did call the fake head request
      mock_head.assert_called()
    @require_tokenizers
    def 						UpperCamelCase_				(  self   )       ->					int:
     # A mock response for an HTTP head request to emulate server down
     SCREAMING_SNAKE_CASE__:   List[str]=      mock.Mock()
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      500
     SCREAMING_SNAKE_CASE__:   Dict=      {}
     SCREAMING_SNAKE_CASE__:   List[str]=      HTTPError
     SCREAMING_SNAKE_CASE__:   Tuple=      {}

     # Download this model to make sure it's in the cache.
     SCREAMING_SNAKE_CASE__:   str=      GPTaTokenizerFast.from_pretrained('''gpt2'''   )

     # Under the mock environment we get a 500 error when trying to reach the tokenizer.
     with mock.patch('''requests.Session.request''' , return_value=lowerCAmelCase   ) as mock_head:
      SCREAMING_SNAKE_CASE__:   int=      GPTaTokenizerFast.from_pretrained('''gpt2'''   )
      # This check we did call the fake head request
      mock_head.assert_called()
    def 						UpperCamelCase_				(  self   )       ->					str:
     # This test is for deprecated behavior and can be removed in v5
     try:
      SCREAMING_SNAKE_CASE__:   List[str]=      tempfile.mktemp()
      with open(lowerCAmelCase , '''wb'''   ) as f:
       http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''' , lowerCAmelCase   )

      SCREAMING_SNAKE_CASE__:   Any=      AlbertTokenizer.from_pretrained(lowerCAmelCase   )
     finally:
      os.remove(lowerCAmelCase   )

     # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
     # the current folder and have the right name.
     if os.path.isfile('''tokenizer.json'''   ):
      # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
      return
     try:
      with open('''tokenizer.json''' , '''wb'''   ) as f:
       http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''' , lowerCAmelCase   )
      SCREAMING_SNAKE_CASE__:   List[str]=      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''   )
      # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
      self.assertEqual(tokenizer.vocab_size , 1000   )
      # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

     finally:
      os.remove('''tokenizer.json'''   )



    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     # This test is for deprecated behavior and can be removed in v5
     SCREAMING_SNAKE_CASE__:   List[str]=      AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   )






@is_staging_test
class 		_lowerCamelCase		(   unittest.TestCase    ):
    __a  =      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]
    @classmethod
    def 						UpperCamelCase_				(  cls   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   str=      TOKEN
     HfFolder.save_token(lowerCAmelCase   )
    @classmethod
    def 						UpperCamelCase_				(  cls   )       ->					List[str]:
     try:
      delete_repo(token=cls._token , repo_id='''test-tokenizer'''   )
     except HTTPError:
      pass

     try:
      delete_repo(token=cls._token , repo_id='''valid_org/test-tokenizer-org'''   )
     except HTTPError:
      pass

     try:
      delete_repo(token=cls._token , repo_id='''test-dynamic-tokenizer'''   )
     except HTTPError:
      pass
    def 						UpperCamelCase_				(  self   )       ->					Any:
     with tempfile.TemporaryDirectory() as tmp_dir:
      SCREAMING_SNAKE_CASE__:   List[Any]=      os.path.join(lowerCAmelCase , '''vocab.txt'''   )
      with open(lowerCAmelCase , '''w''' , encoding='''utf-8'''   ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
      SCREAMING_SNAKE_CASE__:   Tuple=      BertTokenizer(lowerCAmelCase   )

     tokenizer.push_to_hub('''test-tokenizer''' , use_auth_token=self._token   )
     SCREAMING_SNAKE_CASE__:   Optional[int]=      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'   )
     self.assertDictEqual(new_tokenizer.vocab , tokenizer.vocab   )

     # Reset repo
     delete_repo(token=self._token , repo_id='''test-tokenizer'''   )

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
      tokenizer.save_pretrained(lowerCAmelCase , repo_id='''test-tokenizer''' , push_to_hub=lowerCAmelCase , use_auth_token=self._token   )

     SCREAMING_SNAKE_CASE__:   Optional[Any]=      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'   )
     self.assertDictEqual(new_tokenizer.vocab , tokenizer.vocab   )
    def 						UpperCamelCase_				(  self   )       ->					Dict:
     with tempfile.TemporaryDirectory() as tmp_dir:
      SCREAMING_SNAKE_CASE__:   int=      os.path.join(lowerCAmelCase , '''vocab.txt'''   )
      with open(lowerCAmelCase , '''w''' , encoding='''utf-8'''   ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
      SCREAMING_SNAKE_CASE__:   Dict=      BertTokenizer(lowerCAmelCase   )

     tokenizer.push_to_hub('''valid_org/test-tokenizer-org''' , use_auth_token=self._token   )
     SCREAMING_SNAKE_CASE__:   int=      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
     self.assertDictEqual(new_tokenizer.vocab , tokenizer.vocab   )

     # Reset repo
     delete_repo(token=self._token , repo_id='''valid_org/test-tokenizer-org'''   )

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
      tokenizer.save_pretrained(
          lowerCAmelCase , repo_id='''valid_org/test-tokenizer-org''' , push_to_hub=lowerCAmelCase , use_auth_token=self._token   )

     SCREAMING_SNAKE_CASE__:   Optional[int]=      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
     self.assertDictEqual(new_tokenizer.vocab , tokenizer.vocab   )



    @require_tokenizers
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     CustomTokenizer.register_for_auto_class()
     with tempfile.TemporaryDirectory() as tmp_dir:
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      os.path.join(lowerCAmelCase , '''vocab.txt'''   )
      with open(lowerCAmelCase , '''w''' , encoding='''utf-8'''   ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
      SCREAMING_SNAKE_CASE__:   List[str]=      CustomTokenizer(lowerCAmelCase   )

     # No fast custom tokenizer
     tokenizer.push_to_hub('''test-dynamic-tokenizer''' , use_auth_token=self._token   )

     SCREAMING_SNAKE_CASE__:   int=      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer' , trust_remote_code=lowerCAmelCase   )
     # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__ , '''CustomTokenizer'''   )

     # Fast and slow custom tokenizer
     CustomTokenizerFast.register_for_auto_class()
     with tempfile.TemporaryDirectory() as tmp_dir:
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      os.path.join(lowerCAmelCase , '''vocab.txt'''   )
      with open(lowerCAmelCase , '''w''' , encoding='''utf-8'''   ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )

      SCREAMING_SNAKE_CASE__:   Tuple=      BertTokenizerFast.from_pretrained(lowerCAmelCase   )
      bert_tokenizer.save_pretrained(lowerCAmelCase   )
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      CustomTokenizerFast.from_pretrained(lowerCAmelCase   )

     tokenizer.push_to_hub('''test-dynamic-tokenizer''' , use_auth_token=self._token   )

     SCREAMING_SNAKE_CASE__:   str=      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer' , trust_remote_code=lowerCAmelCase   )
     # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__ , '''CustomTokenizerFast'''   )
     SCREAMING_SNAKE_CASE__:   List[Any]=      AutoTokenizer.from_pretrained(
         f'{USER}/test-dynamic-tokenizer' , use_fast=lowerCAmelCase , trust_remote_code=lowerCAmelCase   )
     # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__ , '''CustomTokenizer'''   )






class 		_lowerCamelCase		(   unittest.TestCase    ):
    def 						UpperCamelCase_				(  self   )       ->					int:
     SCREAMING_SNAKE_CASE__:   List[str]=      Trie()
     trie.add('''Hello 友達'''   )
     self.assertEqual(trie.data , {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )
     trie.add('''Hello'''   )
     trie.data
     self.assertEqual(trie.data , {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Any=      Trie()
     self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   ) , ['''[CLS] This is a extra_id_100''']   )
     trie.add('''[CLS]'''   )
     trie.add('''extra_id_1'''   )
     trie.add('''extra_id_100'''   )
     self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   ) , ['''[CLS]''', ''' This is a ''', '''extra_id_100''']   )
    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      Trie()
     trie.add('''A'''   )
     self.assertEqual(trie.split('''ABC'''   ) , ['''A''', '''BC''']   )
     self.assertEqual(trie.split('''BCA'''   ) , ['''BC''', '''A''']   )
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      Trie()
     trie.add('''TOKEN]'''   )
     trie.add('''[SPECIAL_TOKEN]'''   )
     self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   ) , ['''This is something ''', '''[SPECIAL_TOKEN]''']   )
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   Tuple=      Trie()
     trie.add('''A'''   )
     trie.add('''P'''   )
     trie.add('''[SPECIAL_TOKEN]'''   )
     self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   ) , ['''This is something ''', '''[SPECIAL_TOKEN]''']   )
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   Dict=      Trie()
     trie.add('''AB'''   )
     trie.add('''B'''   )
     trie.add('''C'''   )
     self.assertEqual(trie.split('''ABC'''   ) , ['''AB''', '''C''']   )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Optional[Any]=      Trie()
     trie.add('''ABC'''   )
     trie.add('''B'''   )
     trie.add('''CD'''   )
     self.assertEqual(trie.split('''ABCD'''   ) , ['''ABC''', '''D''']   )



    def 						UpperCamelCase_				(  self   )       ->					int:
     # Even if the offsets are wrong, we necessarily output correct string
     # parts.
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      Trie()
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      trie.cut_text('''ABC''' , [0, 0, 2, 1, 2, 3]   )
     self.assertEqual(lowerCAmelCase , ['''AB''', '''C''']   )


from ..utils import DummyObject, requires_backends






class 		_lowerCamelCase		(   metaclass=UpperCamelCase_    ):
    __a  =      ["torch", "scipy"]
    def __init__(  self , *lowerCAmelCase , **lowerCAmelCase   )       ->					Any:
     requires_backends(self , ['''torch''', '''scipy''']   )
    @classmethod
    def 						UpperCamelCase_				(  cls , *lowerCAmelCase , **lowerCAmelCase   )       ->					List[str]:
     requires_backends(cls , ['''torch''', '''scipy''']   )



    @classmethod
    def 						UpperCamelCase_				(  cls , *lowerCAmelCase , **lowerCAmelCase   )       ->					Tuple:
     requires_backends(cls , ['''torch''', '''scipy''']   )

