





'''simple docstring'''


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class  lowerCAmelCase_		(		unittest.TestCase		):




 def 				_snake_case (						self		)     ->				List[Any]:
     # A mock response for an HTTP head request to emulate server down
     _lowerCAmelCase						      =   mock.Mock()
     _lowerCAmelCase						      =   500
     _lowerCAmelCase						      =   {}
     _lowerCAmelCase						      =   HTTPError
     _lowerCAmelCase						      =   {}

     # Download this model to make sure it's in the cache.
     _lowerCAmelCase						      =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)

     # Under the mock environment we get a 500 error when trying to reach the tokenizer.
     with mock.patch("requests.Session.request"      ,  return_value=_lowerCAmelCase		) as mock_head:
         _lowerCAmelCase						      =   BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)
         # This check we did call the fake head request
         mock_head.assert_called()




 @require_tokenizers
 def 				_snake_case (						self		)     ->				Union[str, Any]:
     # A mock response for an HTTP head request to emulate server down
     _lowerCAmelCase						      =   mock.Mock()
     _lowerCAmelCase						      =   500
     _lowerCAmelCase						      =   {}
     _lowerCAmelCase						      =   HTTPError
     _lowerCAmelCase						      =   {}

     # Download this model to make sure it's in the cache.
     _lowerCAmelCase						      =   GPTaTokenizerFast.from_pretrained("gpt2"		)

     # Under the mock environment we get a 500 error when trying to reach the tokenizer.
     with mock.patch("requests.Session.request"      ,  return_value=_lowerCAmelCase		) as mock_head:
         _lowerCAmelCase						      =   GPTaTokenizerFast.from_pretrained("gpt2"		)
         # This check we did call the fake head request
         mock_head.assert_called()




 def 				_snake_case (						self		)     ->				str:
     # This test is for deprecated behavior and can be removed in v5
     try:
         _lowerCAmelCase						      =   tempfile.mktemp()
         with open(_lowerCAmelCase      ,  "wb"		) as f:
             http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"      ,  _lowerCAmelCase		)

         _lowerCAmelCase						      =   AlbertTokenizer.from_pretrained(_lowerCAmelCase		)
     finally:
         os.remove(_lowerCAmelCase		)

     # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
     # the current folder and have the right name.
     if os.path.isfile("tokenizer.json"		):
         # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
         return
     try:
         with open("tokenizer.json"      ,  "wb"		) as f:
             http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"      ,  _lowerCAmelCase		)
         _lowerCAmelCase						      =   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"		)
         # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
         self.assertEqual(tokenizer.vocab_size      ,  1000		)
         # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

     finally:
         os.remove("tokenizer.json"		)





 def 				_snake_case (						self		)     ->				Tuple:
     # This test is for deprecated behavior and can be removed in v5
     _lowerCAmelCase						      =   AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"		)







@is_staging_test
class  lowerCAmelCase_		(		unittest.TestCase		):
 __lowerCamelCase    :				Tuple								=       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




 @classmethod
 def 				_snake_case (						cls		)     ->				Dict:
     _lowerCAmelCase						      =   TOKEN
     HfFolder.save_token(_lowerCAmelCase		)




 @classmethod
 def 				_snake_case (						cls		)     ->				Any:
     try:
         delete_repo(token=cls._token      ,  repo_id="test-tokenizer"		)
     except HTTPError:
         pass

     try:
         delete_repo(token=cls._token      ,  repo_id="valid_org/test-tokenizer-org"		)
     except HTTPError:
         pass

     try:
         delete_repo(token=cls._token      ,  repo_id="test-dynamic-tokenizer"		)
     except HTTPError:
         pass




 def 				_snake_case (						self		)     ->				Tuple:
     with tempfile.TemporaryDirectory() as tmp_dir:
         _lowerCAmelCase						      =   os.path.join(_lowerCAmelCase      ,  "vocab.txt"		)
         with open(_lowerCAmelCase      ,  "w"      ,  encoding="utf-8"		) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
         _lowerCAmelCase						      =   BertTokenizer(_lowerCAmelCase		)

     tokenizer.push_to_hub("test-tokenizer"      ,  use_auth_token=self._token		)
     _lowerCAmelCase						      =   BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''		)
     self.assertDictEqual(new_tokenizer.vocab      ,  tokenizer.vocab		)

     # Reset repo
     delete_repo(token=self._token      ,  repo_id="test-tokenizer"		)

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(_lowerCAmelCase      ,  repo_id="test-tokenizer"      ,  push_to_hub=_lowerCAmelCase      ,  use_auth_token=self._token		)

     _lowerCAmelCase						      =   BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''		)
     self.assertDictEqual(new_tokenizer.vocab      ,  tokenizer.vocab		)




 def 				_snake_case (						self		)     ->				Any:
     with tempfile.TemporaryDirectory() as tmp_dir:
         _lowerCAmelCase						      =   os.path.join(_lowerCAmelCase      ,  "vocab.txt"		)
         with open(_lowerCAmelCase      ,  "w"      ,  encoding="utf-8"		) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
         _lowerCAmelCase						      =   BertTokenizer(_lowerCAmelCase		)

     tokenizer.push_to_hub("valid_org/test-tokenizer-org"      ,  use_auth_token=self._token		)
     _lowerCAmelCase						      =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
     self.assertDictEqual(new_tokenizer.vocab      ,  tokenizer.vocab		)

     # Reset repo
     delete_repo(token=self._token      ,  repo_id="valid_org/test-tokenizer-org"		)

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(
             _lowerCAmelCase      ,  repo_id="valid_org/test-tokenizer-org"      ,  push_to_hub=_lowerCAmelCase      ,  use_auth_token=self._token		)

     _lowerCAmelCase						      =   BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
     self.assertDictEqual(new_tokenizer.vocab      ,  tokenizer.vocab		)





 @require_tokenizers
 def 				_snake_case (						self		)     ->				int:
     CustomTokenizer.register_for_auto_class()
     with tempfile.TemporaryDirectory() as tmp_dir:
         _lowerCAmelCase						      =   os.path.join(_lowerCAmelCase      ,  "vocab.txt"		)
         with open(_lowerCAmelCase      ,  "w"      ,  encoding="utf-8"		) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
         _lowerCAmelCase						      =   CustomTokenizer(_lowerCAmelCase		)

     # No fast custom tokenizer
     tokenizer.push_to_hub("test-dynamic-tokenizer"      ,  use_auth_token=self._token		)

     _lowerCAmelCase						      =   AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''      ,  trust_remote_code=_lowerCAmelCase		)
     # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__      ,  "CustomTokenizer"		)

     # Fast and slow custom tokenizer
     CustomTokenizerFast.register_for_auto_class()
     with tempfile.TemporaryDirectory() as tmp_dir:
         _lowerCAmelCase						      =   os.path.join(_lowerCAmelCase      ,  "vocab.txt"		)
         with open(_lowerCAmelCase      ,  "w"      ,  encoding="utf-8"		) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)

         _lowerCAmelCase						      =   BertTokenizerFast.from_pretrained(_lowerCAmelCase		)
         bert_tokenizer.save_pretrained(_lowerCAmelCase		)
         _lowerCAmelCase						      =   CustomTokenizerFast.from_pretrained(_lowerCAmelCase		)

     tokenizer.push_to_hub("test-dynamic-tokenizer"      ,  use_auth_token=self._token		)

     _lowerCAmelCase						      =   AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''      ,  trust_remote_code=_lowerCAmelCase		)
     # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__      ,  "CustomTokenizerFast"		)
     _lowerCAmelCase						      =   AutoTokenizer.from_pretrained(
         f'''{USER}/test-dynamic-tokenizer'''      ,  use_fast=_lowerCAmelCase      ,  trust_remote_code=_lowerCAmelCase		)
     # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
     self.assertEqual(tokenizer.__class__.__name__      ,  "CustomTokenizer"		)







class  lowerCAmelCase_		(		unittest.TestCase		):




 def 				_snake_case (						self		)     ->				str:
     _lowerCAmelCase						      =   Trie()
     trie.add("Hello 友達"		)
     self.assertEqual(trie.data      ,  {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}		)
     trie.add("Hello"		)
     trie.data
     self.assertEqual(trie.data      ,  {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}		)




 def 				_snake_case (						self		)     ->				Union[str, Any]:
     _lowerCAmelCase						      =   Trie()
     self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)      ,  ["[CLS] This is a extra_id_100"]		)
     trie.add("[CLS]"		)
     trie.add("extra_id_1"		)
     trie.add("extra_id_100"		)
     self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)      ,  ["[CLS]", " This is a ", "extra_id_100"]		)




 def 				_snake_case (						self		)     ->				List[str]:
     _lowerCAmelCase						      =   Trie()
     trie.add("A"		)
     self.assertEqual(trie.split("ABC"		)      ,  ["A", "BC"]		)
     self.assertEqual(trie.split("BCA"		)      ,  ["BC", "A"]		)




 def 				_snake_case (						self		)     ->				Any:
     _lowerCAmelCase						      =   Trie()
     trie.add("TOKEN]"		)
     trie.add("[SPECIAL_TOKEN]"		)
     self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)      ,  ["This is something ", "[SPECIAL_TOKEN]"]		)




 def 				_snake_case (						self		)     ->				Union[str, Any]:
     _lowerCAmelCase						      =   Trie()
     trie.add("A"		)
     trie.add("P"		)
     trie.add("[SPECIAL_TOKEN]"		)
     self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)      ,  ["This is something ", "[SPECIAL_TOKEN]"]		)




 def 				_snake_case (						self		)     ->				List[str]:
     _lowerCAmelCase						      =   Trie()
     trie.add("AB"		)
     trie.add("B"		)
     trie.add("C"		)
     self.assertEqual(trie.split("ABC"		)      ,  ["AB", "C"]		)




 def 				_snake_case (						self		)     ->				List[Any]:
     _lowerCAmelCase						      =   Trie()
     trie.add("ABC"		)
     trie.add("B"		)
     trie.add("CD"		)
     self.assertEqual(trie.split("ABCD"		)      ,  ["ABC", "D"]		)





 def 				_snake_case (						self		)     ->				Any:
     # Even if the offsets are wrong, we necessarily output correct string
     # parts.
     _lowerCAmelCase						      =   Trie()
     _lowerCAmelCase						      =   trie.cut_text("ABC"      ,  [0, 0, 2, 1, 2, 3]		)
     self.assertEqual(_lowerCAmelCase      ,  ["AB", "C"]		)









'''simple docstring'''


from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_SCREAMING_SNAKE_CASE									=				{
    "configuration_mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig"],
    "feature_extraction_mctct": ["MCTCTFeatureExtractor"],
    "processing_mctct": ["MCTCTProcessor"],
}


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _SCREAMING_SNAKE_CASE									=				[
           "MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST",
           "MCTCTForCTC",
           "MCTCTModel",
           "MCTCTPreTrainedModel",
       ]


if TYPE_CHECKING:
       from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
       from .feature_extraction_mctct import MCTCTFeatureExtractor
       from .processing_mctct import MCTCTProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
       import sys

       _SCREAMING_SNAKE_CASE									=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


