



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast







class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   def      UpperCamelCase_  (					self							:			Tuple)				->				Tuple:

    """simple docstring"""







    _snake_case    :  List[str]					=					mock.Mock()
    _snake_case    :  int					=					500
    _snake_case    :  int					=					{}
    _snake_case    :  List[str]					=					HTTPError
    _snake_case    :  Dict					=					{}

    # Download this model to make sure it's in the cache.
    _snake_case    :  List[Any]					=					BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""")

    # Under the mock environment we get a 500 error when trying to reach the tokenizer.
    with mock.patch("""requests.Session.request"""				,		return_value=lowerCAmelCase) as mock_head:
     _snake_case    :  Dict					=					BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""")
     # This check we did call the fake head request
     mock_head.assert_called()

   @require_tokenizers
   def      UpperCamelCase_  (					self							:			int)				->				Tuple:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					mock.Mock()
    _snake_case    :  Dict					=					500
    _snake_case    :  Tuple					=					{}
    _snake_case    :  List[str]					=					HTTPError
    _snake_case    :  Dict					=					{}

    # Download this model to make sure it's in the cache.
    _snake_case    :  int					=					GPTaTokenizerFast.from_pretrained("""gpt2""")

    # Under the mock environment we get a 500 error when trying to reach the tokenizer.
    with mock.patch("""requests.Session.request"""				,		return_value=lowerCAmelCase) as mock_head:
     _snake_case    :  List[Any]					=					GPTaTokenizerFast.from_pretrained("""gpt2""")
     # This check we did call the fake head request
     mock_head.assert_called()

   def      UpperCamelCase_  (					self							:			Dict)				->				Tuple:

    """simple docstring"""







    try:
     _snake_case    :  Optional[int]					=					tempfile.mktemp()
     with open(lowerCAmelCase				,		"""wb""") as f:
      http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""				,		lowerCAmelCase)

     _snake_case    :  Optional[Any]					=					AlbertTokenizer.from_pretrained(lowerCAmelCase)
    finally:
     os.remove(lowerCAmelCase)

    # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
    # the current folder and have the right name.
    if os.path.isfile("""tokenizer.json"""):
     # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
     return
    try:
     with open("""tokenizer.json"""				,		"""wb""") as f:
      http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""				,		lowerCAmelCase)
     _snake_case    :  List[Any]					=					AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2""")
     # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
     self.assertEqual(tokenizer.vocab_size				,		1000)
     # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

    finally:
     os.remove("""tokenizer.json""")


   def      UpperCamelCase_  (					self							:			List[str])				->				Dict:

    """simple docstring"""







    _snake_case    :  str					=					AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model""")







@is_staging_test
class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   snake_case_					: Optional[int]          =		["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

   @classmethod
   def      UpperCamelCase_  (					cls							:			Union[str, Any])				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  str					=					TOKEN
    HfFolder.save_token(lowerCAmelCase)

   @classmethod
   def      UpperCamelCase_  (					cls							:			str)				->				List[Any]:

    """simple docstring"""







    try:
     delete_repo(token=cls._token				,		repo_id="""test-tokenizer""")
    except HTTPError:
     pass

    try:
     delete_repo(token=cls._token				,		repo_id="""valid_org/test-tokenizer-org""")
    except HTTPError:
     pass

    try:
     delete_repo(token=cls._token				,		repo_id="""test-dynamic-tokenizer""")
    except HTTPError:
     pass

   def      UpperCamelCase_  (					self							:			int)				->				int:

    """simple docstring"""







    with tempfile.TemporaryDirectory() as tmp_dir:
     _snake_case    :  List[Any]					=					os.path.join(lowerCAmelCase				,		"""vocab.txt""")
     with open(lowerCAmelCase				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
      vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
     _snake_case    :  Union[str, Any]					=					BertTokenizer(lowerCAmelCase)

    tokenizer.push_to_hub("""test-tokenizer"""				,		use_auth_token=self._token)
    _snake_case    :  Tuple					=					BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer''')
    self.assertDictEqual(new_tokenizer.vocab				,		tokenizer.vocab)

    # Reset repo
    delete_repo(token=self._token				,		repo_id="""test-tokenizer""")

    # Push to hub via save_pretrained
    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer.save_pretrained(lowerCAmelCase				,		repo_id="""test-tokenizer"""				,		push_to_hub=lowerCAmelCase				,		use_auth_token=self._token)

    _snake_case    :  int					=					BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer''')
    self.assertDictEqual(new_tokenizer.vocab				,		tokenizer.vocab)

   def      UpperCamelCase_  (					self							:			str)				->				Union[str, Any]:

    """simple docstring"""







    with tempfile.TemporaryDirectory() as tmp_dir:
     _snake_case    :  List[str]					=					os.path.join(lowerCAmelCase				,		"""vocab.txt""")
     with open(lowerCAmelCase				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
      vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
     _snake_case    :  Union[str, Any]					=					BertTokenizer(lowerCAmelCase)

    tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""				,		use_auth_token=self._token)
    _snake_case    :  List[str]					=					BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""")
    self.assertDictEqual(new_tokenizer.vocab				,		tokenizer.vocab)

    # Reset repo
    delete_repo(token=self._token				,		repo_id="""valid_org/test-tokenizer-org""")

    # Push to hub via save_pretrained
    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer.save_pretrained(
         lowerCAmelCase				,		repo_id="""valid_org/test-tokenizer-org"""				,		push_to_hub=lowerCAmelCase				,		use_auth_token=self._token)

    _snake_case    :  List[str]					=					BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""")
    self.assertDictEqual(new_tokenizer.vocab				,		tokenizer.vocab)


   @require_tokenizers
   def      UpperCamelCase_  (					self							:			List[str])				->				Any:

    """simple docstring"""







    CustomTokenizer.register_for_auto_class()
    with tempfile.TemporaryDirectory() as tmp_dir:
     _snake_case    :  str					=					os.path.join(lowerCAmelCase				,		"""vocab.txt""")
     with open(lowerCAmelCase				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
      vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
     _snake_case    :  List[str]					=					CustomTokenizer(lowerCAmelCase)

    # No fast custom tokenizer
    tokenizer.push_to_hub("""test-dynamic-tokenizer"""				,		use_auth_token=self._token)

    _snake_case    :  Union[str, Any]					=					AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''				,		trust_remote_code=lowerCAmelCase)
    # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
    self.assertEqual(tokenizer.__class__.__name__				,		"""CustomTokenizer""")

    # Fast and slow custom tokenizer
    CustomTokenizerFast.register_for_auto_class()
    with tempfile.TemporaryDirectory() as tmp_dir:
     _snake_case    :  Optional[Any]					=					os.path.join(lowerCAmelCase				,		"""vocab.txt""")
     with open(lowerCAmelCase				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
      vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))

     _snake_case    :  List[Any]					=					BertTokenizerFast.from_pretrained(lowerCAmelCase)
     bert_tokenizer.save_pretrained(lowerCAmelCase)
     _snake_case    :  Dict					=					CustomTokenizerFast.from_pretrained(lowerCAmelCase)

    tokenizer.push_to_hub("""test-dynamic-tokenizer"""				,		use_auth_token=self._token)

    _snake_case    :  List[Any]					=					AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''				,		trust_remote_code=lowerCAmelCase)
    # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
    self.assertEqual(tokenizer.__class__.__name__				,		"""CustomTokenizerFast""")
    _snake_case    :  Optional[int]					=					AutoTokenizer.from_pretrained(
        F'''{USER}/test-dynamic-tokenizer'''				,		use_fast=lowerCAmelCase				,		trust_remote_code=lowerCAmelCase)
    # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
    self.assertEqual(tokenizer.__class__.__name__				,		"""CustomTokenizer""")







class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   def      UpperCamelCase_  (					self							:			Dict)				->				Union[str, Any]:

    """simple docstring"""







    _snake_case    :  Dict					=					Trie()
    trie.add("""Hello 友達""")
    self.assertEqual(trie.data				,		{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}})
    trie.add("""Hello""")
    trie.data
    self.assertEqual(trie.data				,		{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}})

   def      UpperCamelCase_  (					self							:			str)				->				Optional[Any]:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					Trie()
    self.assertEqual(trie.split("""[CLS] This is a extra_id_100""")				,		["""[CLS] This is a extra_id_100"""])
    trie.add("""[CLS]""")
    trie.add("""extra_id_1""")
    trie.add("""extra_id_100""")
    self.assertEqual(trie.split("""[CLS] This is a extra_id_100""")				,		["""[CLS]""", """ This is a """, """extra_id_100"""])

   def      UpperCamelCase_  (					self							:			str)				->				List[str]:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					Trie()
    trie.add("""A""")
    self.assertEqual(trie.split("""ABC""")				,		["""A""", """BC"""])
    self.assertEqual(trie.split("""BCA""")				,		["""BC""", """A"""])

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				List[Any]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					Trie()
    trie.add("""TOKEN]""")
    trie.add("""[SPECIAL_TOKEN]""")
    self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""")				,		["""This is something """, """[SPECIAL_TOKEN]"""])

   def      UpperCamelCase_  (					self							:			Optional[int])				->				Tuple:

    """simple docstring"""







    _snake_case    :  Tuple					=					Trie()
    trie.add("""A""")
    trie.add("""P""")
    trie.add("""[SPECIAL_TOKEN]""")
    self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""")				,		["""This is something """, """[SPECIAL_TOKEN]"""])

   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				List[str]:

    """simple docstring"""







    _snake_case    :  Optional[int]					=					Trie()
    trie.add("""AB""")
    trie.add("""B""")
    trie.add("""C""")
    self.assertEqual(trie.split("""ABC""")				,		["""AB""", """C"""])

   def      UpperCamelCase_  (					self							:			Any)				->				str:

    """simple docstring"""







    _snake_case    :  List[Any]					=					Trie()
    trie.add("""ABC""")
    trie.add("""B""")
    trie.add("""CD""")
    self.assertEqual(trie.split("""ABCD""")				,		["""ABC""", """D"""])


   def      UpperCamelCase_  (					self							:			int)				->				Dict:

    """simple docstring"""







    _snake_case    :  Any					=					Trie()
    _snake_case    :  str					=					trie.cut_text("""ABC"""				,		[0, 0, 2, 1, 2, 3])
    self.assertEqual(lowerCAmelCase				,		["""AB""", """C"""])





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a__    =	{
    """configuration_canine""": ["""CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CanineConfig"""],
    """tokenization_canine""": ["""CanineTokenizer"""],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a__    =	[
      """CANINE_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """CanineForMultipleChoice""",
      """CanineForQuestionAnswering""",
      """CanineForSequenceClassification""",
      """CanineForTokenClassification""",
      """CanineLayer""",
      """CanineModel""",
      """CaninePreTrainedModel""",
      """load_tf_weights_in_canine""",
  ]


if TYPE_CHECKING:
  from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
  from .tokenization_canine import CanineTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_canine import (
        CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
        CanineForMultipleChoice,
        CanineForQuestionAnswering,
        CanineForSequenceClassification,
        CanineForTokenClassification,
        CanineLayer,
        CanineModel,
        CaninePreTrainedModel,
        load_tf_weights_in_canine,
    )


else:
  import sys

  a__    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
