



"""simple docstring"""
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast


class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):

    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= mock.Mock()
      lowercase_								= 5_0_0
      lowercase_								= {}
      lowercase_								= HTTPError
      lowercase_								= {}

      # Download this model to make sure it's in the cache.
      lowercase_								= BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""")

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("""requests.Session.request""" ,  return_value=lowerCAmelCase_) as mock_head:
        lowercase_								= BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""")
        # This check we did call the fake head request
        mock_head.assert_called()

    @require_tokenizers
    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= mock.Mock()
      lowercase_								= 5_0_0
      lowercase_								= {}
      lowercase_								= HTTPError
      lowercase_								= {}

      # Download this model to make sure it's in the cache.
      lowercase_								= GPTaTokenizerFast.from_pretrained("""gpt2""")

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("""requests.Session.request""" ,  return_value=lowerCAmelCase_) as mock_head:
        lowercase_								= GPTaTokenizerFast.from_pretrained("""gpt2""")
        # This check we did call the fake head request
        mock_head.assert_called()

    def 					_UpperCAmelCase	(				self			:				Any):







      """simple docstring"""







      try:
        lowercase_								= tempfile.mktemp()
        with open(lowerCAmelCase_ ,  """wb""") as f:
          http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model""" ,  lowerCAmelCase_)

        lowercase_								= AlbertTokenizer.from_pretrained(lowerCAmelCase_)
      finally:
        os.remove(lowerCAmelCase_)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile("""tokenizer.json"""):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
      try:
        with open("""tokenizer.json""" ,  """wb""") as f:
          http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json""" ,  lowerCAmelCase_)
        lowercase_								= AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2""")
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size ,  1_0_0_0)
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
        os.remove("""tokenizer.json""")




    def 					_UpperCAmelCase	(				self			:				int):







      """simple docstring"""







      lowercase_								= AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model""")







@is_staging_test
class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):
    lowercase__      =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

    @classmethod
    def 					_UpperCAmelCase	(				cls			:				Dict):







      """simple docstring"""







      lowercase_								= TOKEN
      HfFolder.save_token(lowerCAmelCase_)

    @classmethod
    def 					_UpperCAmelCase	(				cls			:				Optional[Any]):







      """simple docstring"""







      try:
        delete_repo(token=cls._token ,  repo_id="""test-tokenizer""")
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token ,  repo_id="""valid_org/test-tokenizer-org""")
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token ,  repo_id="""test-dynamic-tokenizer""")
      except HTTPError:
        pass

    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase_								= os.path.join(lowerCAmelCase_ ,  """vocab.txt""")
        with open(lowerCAmelCase_ ,  """w""" ,  encoding="""utf-8""") as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
        lowercase_								= BertTokenizer(lowerCAmelCase_)

      tokenizer.push_to_hub("""test-tokenizer""" ,  use_auth_token=self._token)
      lowercase_								= BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer''')
      self.assertDictEqual(new_tokenizer.vocab ,  tokenizer.vocab)

      # Reset repo
      delete_repo(token=self._token ,  repo_id="""test-tokenizer""")

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(lowerCAmelCase_ ,  repo_id="""test-tokenizer""" ,  push_to_hub=lowerCAmelCase_ ,  use_auth_token=self._token)

      lowercase_								= BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer''')
      self.assertDictEqual(new_tokenizer.vocab ,  tokenizer.vocab)

    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase_								= os.path.join(lowerCAmelCase_ ,  """vocab.txt""")
        with open(lowerCAmelCase_ ,  """w""" ,  encoding="""utf-8""") as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
        lowercase_								= BertTokenizer(lowerCAmelCase_)

      tokenizer.push_to_hub("""valid_org/test-tokenizer-org""" ,  use_auth_token=self._token)
      lowercase_								= BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""")
      self.assertDictEqual(new_tokenizer.vocab ,  tokenizer.vocab)

      # Reset repo
      delete_repo(token=self._token ,  repo_id="""valid_org/test-tokenizer-org""")

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            lowerCAmelCase_ ,  repo_id="""valid_org/test-tokenizer-org""" ,  push_to_hub=lowerCAmelCase_ ,  use_auth_token=self._token)

      lowercase_								= BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""")
      self.assertDictEqual(new_tokenizer.vocab ,  tokenizer.vocab)




    @require_tokenizers
    def 					_UpperCAmelCase	(				self			:				Any):







      """simple docstring"""







      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase_								= os.path.join(lowerCAmelCase_ ,  """vocab.txt""")
        with open(lowerCAmelCase_ ,  """w""" ,  encoding="""utf-8""") as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
        lowercase_								= CustomTokenizer(lowerCAmelCase_)

      # No fast custom tokenizer
      tokenizer.push_to_hub("""test-dynamic-tokenizer""" ,  use_auth_token=self._token)

      lowercase_								= AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''' ,  trust_remote_code=lowerCAmelCase_)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__ ,  """CustomTokenizer""")

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase_								= os.path.join(lowerCAmelCase_ ,  """vocab.txt""")
        with open(lowerCAmelCase_ ,  """w""" ,  encoding="""utf-8""") as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))

        lowercase_								= BertTokenizerFast.from_pretrained(lowerCAmelCase_)
        bert_tokenizer.save_pretrained(lowerCAmelCase_)
        lowercase_								= CustomTokenizerFast.from_pretrained(lowerCAmelCase_)

      tokenizer.push_to_hub("""test-dynamic-tokenizer""" ,  use_auth_token=self._token)

      lowercase_								= AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''' ,  trust_remote_code=lowerCAmelCase_)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__ ,  """CustomTokenizerFast""")
      lowercase_								= AutoTokenizer.from_pretrained(
          F'''{USER}/test-dynamic-tokenizer''' ,  use_fast=lowerCAmelCase_ ,  trust_remote_code=lowerCAmelCase_)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__ ,  """CustomTokenizer""")







class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      lowercase_								= Trie()
      trie.add("""Hello 友達""")
      self.assertEqual(trie.data ,  {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}})
      trie.add("""Hello""")
      trie.data
      self.assertEqual(trie.data ,  {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}})

    def 					_UpperCAmelCase	(				self			:				List[Any]):







      """simple docstring"""







      lowercase_								= Trie()
      self.assertEqual(trie.split("""[CLS] This is a extra_id_100""") ,  ["""[CLS] This is a extra_id_100"""])
      trie.add("""[CLS]""")
      trie.add("""extra_id_1""")
      trie.add("""extra_id_100""")
      self.assertEqual(trie.split("""[CLS] This is a extra_id_100""") ,  ["""[CLS]""", """ This is a """, """extra_id_100"""])

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      lowercase_								= Trie()
      trie.add("""A""")
      self.assertEqual(trie.split("""ABC""") ,  ["""A""", """BC"""])
      self.assertEqual(trie.split("""BCA""") ,  ["""BC""", """A"""])

    def 					_UpperCAmelCase	(				self			:				Any):







      """simple docstring"""







      lowercase_								= Trie()
      trie.add("""TOKEN]""")
      trie.add("""[SPECIAL_TOKEN]""")
      self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""") ,  ["""This is something """, """[SPECIAL_TOKEN]"""])

    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= Trie()
      trie.add("""A""")
      trie.add("""P""")
      trie.add("""[SPECIAL_TOKEN]""")
      self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""") ,  ["""This is something """, """[SPECIAL_TOKEN]"""])

    def 					_UpperCAmelCase	(				self			:				str):







      """simple docstring"""







      lowercase_								= Trie()
      trie.add("""AB""")
      trie.add("""B""")
      trie.add("""C""")
      self.assertEqual(trie.split("""ABC""") ,  ["""AB""", """C"""])

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      lowercase_								= Trie()
      trie.add("""ABC""")
      trie.add("""B""")
      trie.add("""CD""")
      self.assertEqual(trie.split("""ABCD""") ,  ["""ABC""", """D"""])




    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      lowercase_								= Trie()
      lowercase_								= trie.cut_text("""ABC""" ,  [0, 0, 2, 1, 2, 3])
      self.assertEqual(lowerCAmelCase_ ,  ["""AB""", """C"""])






"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import _LazyModule


UpperCAmelCase						:		List[Any]			= {"tokenization_bertweet": ["BertweetTokenizer"]}


if TYPE_CHECKING:
       from .tokenization_bertweet import BertweetTokenizer

else:
       import sys

       UpperCAmelCase						:		Tuple			= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

