






"""simple docstring"""


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast






class 		snake_case						(						unittest.TestCase ):





    def        lowerCamelCase__   (							self			:      str    ):


      '''simple docstring'''




      a  :    Optional[int]       =     mock.Mock()
      a  :    int       =     5_0_0
      a  :    Dict       =     {}
      a  :    Dict       =     HTTPError
      a  :    Dict       =     {}

      # Download this model to make sure it's in the cache.
      a  :    List[Any]       =     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('requests.Session.request'       ,	return_value=A    ) as mock_head:
        a  :    Tuple       =     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )
        # This check we did call the fake head request
        mock_head.assert_called()





    @require_tokenizers
    def        lowerCamelCase__   (							self			:      List[Any]    ):


      '''simple docstring'''




      a  :    str       =     mock.Mock()
      a  :    Tuple       =     5_0_0
      a  :    Any       =     {}
      a  :    Optional[Any]       =     HTTPError
      a  :    Any       =     {}

      # Download this model to make sure it's in the cache.
      a  :    str       =     GPTaTokenizerFast.from_pretrained('gpt2'    )

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('requests.Session.request'       ,	return_value=A    ) as mock_head:
        a  :    Any       =     GPTaTokenizerFast.from_pretrained('gpt2'    )
        # This check we did call the fake head request
        mock_head.assert_called()





    def        lowerCamelCase__   (							self			:      List[str]    ):


      '''simple docstring'''




      try:
        a  :    Any       =     tempfile.mktemp()
        with open(A       ,	'wb'    ) as f:
          http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       ,	A    )

        a  :    Union[str, Any]       =     AlbertTokenizer.from_pretrained(A    )
      finally:
        os.remove(A    )

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('tokenizer.json'    ):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
      try:
        with open('tokenizer.json'       ,	'wb'    ) as f:
          http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'       ,	A    )
        a  :    Union[str, Any]       =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'    )
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size       ,	1_0_0_0    )
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
        os.remove('tokenizer.json'    )





    def        lowerCamelCase__   (							self			:      Dict    ):


      '''simple docstring'''




      a  :    Union[str, Any]       =     AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'    )






@is_staging_test
class 		snake_case						(						unittest.TestCase ):
    __magic_name__					  =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





    @classmethod
    def        lowerCamelCase__   (							cls			:      List[Any]    ):


      '''simple docstring'''




      a  :    Optional[Any]       =     TOKEN
      HfFolder.save_token(A    )





    @classmethod
    def        lowerCamelCase__   (							cls			:      Tuple    ):


      '''simple docstring'''




      try:
        delete_repo(token=cls._token       ,	repo_id='test-tokenizer'    )
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token       ,	repo_id='valid_org/test-tokenizer-org'    )
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token       ,	repo_id='test-dynamic-tokenizer'    )
      except HTTPError:
        pass





    def        lowerCamelCase__   (							self			:      List[Any]    ):


      '''simple docstring'''




      with tempfile.TemporaryDirectory() as tmp_dir:
        a  :    Tuple       =     os.path.join(A       ,	'vocab.txt'    )
        with open(A       ,	'w'       ,	encoding='utf-8'    ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
        a  :    List[Any]       =     BertTokenizer(A    )

      tokenizer.push_to_hub('test-tokenizer'       ,	use_auth_token=self._token    )
      a  :    List[Any]       =     BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''    )
      self.assertDictEqual(new_tokenizer.vocab       ,	tokenizer.vocab    )

      # Reset repo
      delete_repo(token=self._token       ,	repo_id='test-tokenizer'    )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(A       ,	repo_id='test-tokenizer'       ,	push_to_hub=A       ,	use_auth_token=self._token    )

      a  :    Optional[Any]       =     BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''    )
      self.assertDictEqual(new_tokenizer.vocab       ,	tokenizer.vocab    )





    def        lowerCamelCase__   (							self			:      Union[str, Any]    ):


      '''simple docstring'''




      with tempfile.TemporaryDirectory() as tmp_dir:
        a  :    Optional[int]       =     os.path.join(A       ,	'vocab.txt'    )
        with open(A       ,	'w'       ,	encoding='utf-8'    ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
        a  :    int       =     BertTokenizer(A    )

      tokenizer.push_to_hub('valid_org/test-tokenizer-org'       ,	use_auth_token=self._token    )
      a  :    int       =     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'    )
      self.assertDictEqual(new_tokenizer.vocab       ,	tokenizer.vocab    )

      # Reset repo
      delete_repo(token=self._token       ,	repo_id='valid_org/test-tokenizer-org'    )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            A       ,	repo_id='valid_org/test-tokenizer-org'       ,	push_to_hub=A       ,	use_auth_token=self._token    )

      a  :    List[Any]       =     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'    )
      self.assertDictEqual(new_tokenizer.vocab       ,	tokenizer.vocab    )





    @require_tokenizers
    def        lowerCamelCase__   (							self			:      List[str]    ):


      '''simple docstring'''




      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        a  :    int       =     os.path.join(A       ,	'vocab.txt'    )
        with open(A       ,	'w'       ,	encoding='utf-8'    ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
        a  :    Union[str, Any]       =     CustomTokenizer(A    )

      # No fast custom tokenizer
      tokenizer.push_to_hub('test-dynamic-tokenizer'       ,	use_auth_token=self._token    )

      a  :    str       =     AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''       ,	trust_remote_code=A    )
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__       ,	'CustomTokenizer'    )

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        a  :    Optional[Any]       =     os.path.join(A       ,	'vocab.txt'    )
        with open(A       ,	'w'       ,	encoding='utf-8'    ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )

        a  :    str       =     BertTokenizerFast.from_pretrained(A    )
        bert_tokenizer.save_pretrained(A    )
        a  :    List[str]       =     CustomTokenizerFast.from_pretrained(A    )

      tokenizer.push_to_hub('test-dynamic-tokenizer'       ,	use_auth_token=self._token    )

      a  :    str       =     AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''       ,	trust_remote_code=A    )
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__       ,	'CustomTokenizerFast'    )
      a  :    str       =     AutoTokenizer.from_pretrained(
          F'''{USER}/test-dynamic-tokenizer'''       ,	use_fast=A       ,	trust_remote_code=A    )
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__       ,	'CustomTokenizer'    )






class 		snake_case						(						unittest.TestCase ):





    def        lowerCamelCase__   (							self			:      Optional[Any]    ):


      '''simple docstring'''




      a  :    int       =     Trie()
      trie.add('Hello 友達'    )
      self.assertEqual(trie.data       ,	{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}    )
      trie.add('Hello'    )
      trie.data
      self.assertEqual(trie.data       ,	{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}    )





    def        lowerCamelCase__   (							self			:      List[Any]    ):


      '''simple docstring'''




      a  :    List[str]       =     Trie()
      self.assertEqual(trie.split('[CLS] This is a extra_id_100'    )       ,	['[CLS] This is a extra_id_100']    )
      trie.add('[CLS]'    )
      trie.add('extra_id_1'    )
      trie.add('extra_id_100'    )
      self.assertEqual(trie.split('[CLS] This is a extra_id_100'    )       ,	['[CLS]', ' This is a ', 'extra_id_100']    )





    def        lowerCamelCase__   (							self			:      List[str]    ):


      '''simple docstring'''




      a  :    Tuple       =     Trie()
      trie.add('A'    )
      self.assertEqual(trie.split('ABC'    )       ,	['A', 'BC']    )
      self.assertEqual(trie.split('BCA'    )       ,	['BC', 'A']    )





    def        lowerCamelCase__   (							self			:      List[Any]    ):


      '''simple docstring'''




      a  :    List[str]       =     Trie()
      trie.add('TOKEN]'    )
      trie.add('[SPECIAL_TOKEN]'    )
      self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'    )       ,	['This is something ', '[SPECIAL_TOKEN]']    )





    def        lowerCamelCase__   (							self			:      Tuple    ):


      '''simple docstring'''




      a  :    int       =     Trie()
      trie.add('A'    )
      trie.add('P'    )
      trie.add('[SPECIAL_TOKEN]'    )
      self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'    )       ,	['This is something ', '[SPECIAL_TOKEN]']    )





    def        lowerCamelCase__   (							self			:      Any    ):


      '''simple docstring'''




      a  :    Optional[Any]       =     Trie()
      trie.add('AB'    )
      trie.add('B'    )
      trie.add('C'    )
      self.assertEqual(trie.split('ABC'    )       ,	['AB', 'C']    )





    def        lowerCamelCase__   (							self			:      List[str]    ):


      '''simple docstring'''




      a  :    Optional[Any]       =     Trie()
      trie.add('ABC'    )
      trie.add('B'    )
      trie.add('CD'    )
      self.assertEqual(trie.split('ABCD'    )       ,	['ABC', 'D']    )





    def        lowerCamelCase__   (							self			:      List[str]    ):


      '''simple docstring'''




      a  :    int       =     Trie()
      a  :    List[Any]       =     trie.cut_text('ABC'       ,	[0, 0, 2, 1, 2, 3]    )
      self.assertEqual(A       ,	['AB', 'C']    )












"""simple docstring"""


from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCamelCase       :					Optional[Any]									=		{
    'configuration_informer': [
        'INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'InformerConfig',
    ],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _UpperCamelCase       :					Tuple									=		[
       'INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
       'InformerForPrediction',
       'InformerModel',
       'InformerPreTrainedModel',
   ]


if TYPE_CHECKING:
   from .configuration_informer import INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, InformerConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_informer import (
          INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          InformerForPrediction,
          InformerModel,
          InformerPreTrainedModel,
      )

else:
   import sys

   _UpperCamelCase       :					List[str]									=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




