

'''simple docstring'''



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast



class 				_snake_case					(   unittest.TestCase			):

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							mock.Mock()
      lowerCAmelCase 						=							5_00
      lowerCAmelCase 						=							{}
      lowerCAmelCase 						=							HTTPError
      lowerCAmelCase 						=							{}

      # Download this model to make sure it's in the cache.
      lowerCAmelCase 						=							BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'	)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('requests.Session.request'      , return_value=_SCREAMING_SNAKE_CASE	) as mock_head:
          lowerCAmelCase 						=							BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'	)
          # This check we did call the fake head request
          mock_head.assert_called()

  @require_tokenizers
  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							mock.Mock()
      lowerCAmelCase 						=							5_00
      lowerCAmelCase 						=							{}
      lowerCAmelCase 						=							HTTPError
      lowerCAmelCase 						=							{}

      # Download this model to make sure it's in the cache.
      lowerCAmelCase 						=							GPTaTokenizerFast.from_pretrained('gpt2'	)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('requests.Session.request'      , return_value=_SCREAMING_SNAKE_CASE	) as mock_head:
          lowerCAmelCase 						=							GPTaTokenizerFast.from_pretrained('gpt2'	)
          # This check we did call the fake head request
          mock_head.assert_called()

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      try:
          lowerCAmelCase 						=							tempfile.mktemp()
          with open(_SCREAMING_SNAKE_CASE      , 'wb'	) as f:
              http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'      , _SCREAMING_SNAKE_CASE	)

          lowerCAmelCase 						=							AlbertTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE	)
      finally:
          os.remove(_SCREAMING_SNAKE_CASE	)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('tokenizer.json'	):
          # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
          return
      try:
          with open('tokenizer.json'      , 'wb'	) as f:
              http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'      , _SCREAMING_SNAKE_CASE	)
          lowerCAmelCase 						=							AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'	)
          # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
          self.assertEqual(tokenizer.vocab_size      , 10_00	)
          # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
          os.remove('tokenizer.json'	)


  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'	)



@is_staging_test
class 				_snake_case					(   unittest.TestCase			):
  SCREAMING_SNAKE_CASE			:       Optional[Any]				=     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']

  @classmethod
  def   _SCREAMING_SNAKE_CASE				(				cls	):





      '''simple docstring'''

      lowerCAmelCase 						=							TOKEN
      HfFolder.save_token(_SCREAMING_SNAKE_CASE	)

  @classmethod
  def   _SCREAMING_SNAKE_CASE				(				cls	):





      '''simple docstring'''

      try:
          delete_repo(token=cls._token      , repo_id='test-tokenizer'	)
      except HTTPError:
          pass

      try:
          delete_repo(token=cls._token      , repo_id='valid_org/test-tokenizer-org'	)
      except HTTPError:
          pass

      try:
          delete_repo(token=cls._token      , repo_id='test-dynamic-tokenizer'	)
      except HTTPError:
          pass

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase 						=							os.path.join(_SCREAMING_SNAKE_CASE      , 'vocab.txt'	)
          with open(_SCREAMING_SNAKE_CASE      , 'w'      , encoding='utf-8'	) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]	)	)
          lowerCAmelCase 						=							BertTokenizer(_SCREAMING_SNAKE_CASE	)

      tokenizer.push_to_hub('test-tokenizer'      , use_auth_token=self._token	)
      lowerCAmelCase 						=							BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'	)
      self.assertDictEqual(new_tokenizer.vocab      , tokenizer.vocab	)

      # Reset repo
      delete_repo(token=self._token      , repo_id='test-tokenizer'	)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE      , repo_id='test-tokenizer'      , push_to_hub=_SCREAMING_SNAKE_CASE      , use_auth_token=self._token	)

      lowerCAmelCase 						=							BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'	)
      self.assertDictEqual(new_tokenizer.vocab      , tokenizer.vocab	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase 						=							os.path.join(_SCREAMING_SNAKE_CASE      , 'vocab.txt'	)
          with open(_SCREAMING_SNAKE_CASE      , 'w'      , encoding='utf-8'	) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]	)	)
          lowerCAmelCase 						=							BertTokenizer(_SCREAMING_SNAKE_CASE	)

      tokenizer.push_to_hub('valid_org/test-tokenizer-org'      , use_auth_token=self._token	)
      lowerCAmelCase 						=							BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'	)
      self.assertDictEqual(new_tokenizer.vocab      , tokenizer.vocab	)

      # Reset repo
      delete_repo(token=self._token      , repo_id='valid_org/test-tokenizer-org'	)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(
              _SCREAMING_SNAKE_CASE      , repo_id='valid_org/test-tokenizer-org'      , push_to_hub=_SCREAMING_SNAKE_CASE      , use_auth_token=self._token	)

      lowerCAmelCase 						=							BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'	)
      self.assertDictEqual(new_tokenizer.vocab      , tokenizer.vocab	)


  @require_tokenizers
  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase 						=							os.path.join(_SCREAMING_SNAKE_CASE      , 'vocab.txt'	)
          with open(_SCREAMING_SNAKE_CASE      , 'w'      , encoding='utf-8'	) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]	)	)
          lowerCAmelCase 						=							CustomTokenizer(_SCREAMING_SNAKE_CASE	)

      # No fast custom tokenizer
      tokenizer.push_to_hub('test-dynamic-tokenizer'      , use_auth_token=self._token	)

      lowerCAmelCase 						=							AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'      , trust_remote_code=_SCREAMING_SNAKE_CASE	)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__      , 'CustomTokenizer'	)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase 						=							os.path.join(_SCREAMING_SNAKE_CASE      , 'vocab.txt'	)
          with open(_SCREAMING_SNAKE_CASE      , 'w'      , encoding='utf-8'	) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]	)	)

          lowerCAmelCase 						=							BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE	)
          bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE	)
          lowerCAmelCase 						=							CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE	)

      tokenizer.push_to_hub('test-dynamic-tokenizer'      , use_auth_token=self._token	)

      lowerCAmelCase 						=							AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'      , trust_remote_code=_SCREAMING_SNAKE_CASE	)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__      , 'CustomTokenizerFast'	)
      lowerCAmelCase 						=							AutoTokenizer.from_pretrained(
          F'{USER}/test-dynamic-tokenizer'      , use_fast=_SCREAMING_SNAKE_CASE      , trust_remote_code=_SCREAMING_SNAKE_CASE	)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__      , 'CustomTokenizer'	)



class 				_snake_case					(   unittest.TestCase			):

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      trie.add('Hello 友達'	)
      self.assertEqual(trie.data      , {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}	)
      trie.add('Hello'	)
      trie.data
      self.assertEqual(trie.data      , {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      self.assertEqual(trie.split('[CLS] This is a extra_id_100'	)      , ['[CLS] This is a extra_id_100']	)
      trie.add('[CLS]'	)
      trie.add('extra_id_1'	)
      trie.add('extra_id_100'	)
      self.assertEqual(trie.split('[CLS] This is a extra_id_100'	)      , ['[CLS]', ' This is a ', 'extra_id_100']	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      trie.add('A'	)
      self.assertEqual(trie.split('ABC'	)      , ['A', 'BC']	)
      self.assertEqual(trie.split('BCA'	)      , ['BC', 'A']	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      trie.add('TOKEN]'	)
      trie.add('[SPECIAL_TOKEN]'	)
      self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'	)      , ['This is something ', '[SPECIAL_TOKEN]']	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      trie.add('A'	)
      trie.add('P'	)
      trie.add('[SPECIAL_TOKEN]'	)
      self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'	)      , ['This is something ', '[SPECIAL_TOKEN]']	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      trie.add('AB'	)
      trie.add('B'	)
      trie.add('C'	)
      self.assertEqual(trie.split('ABC'	)      , ['AB', 'C']	)

  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      trie.add('ABC'	)
      trie.add('B'	)
      trie.add('CD'	)
      self.assertEqual(trie.split('ABCD'	)      , ['ABC', 'D']	)


  def   _SCREAMING_SNAKE_CASE				(				self	):





      '''simple docstring'''

      lowerCAmelCase 						=							Trie()
      lowerCAmelCase 						=							trie.cut_text('ABC'      , [0, 0, 2, 1, 2, 3]	)
      self.assertEqual(_SCREAMING_SNAKE_CASE      , ['AB', 'C']	)








'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCamelCase						:  Optional[int]							       =				logging.get_logger(__name__)

_UpperCamelCase						:  Tuple							       =				{
    "google/canine-s": "https://huggingface.co/google/canine-s/resolve/main/config.json",
    # See all CANINE models at https://huggingface.co/models?filter=canine
}



class 				_snake_case					(   a_			):
  SCREAMING_SNAKE_CASE			:       List[Any]				=     '''canine'''

  def __init__(				self      , _SCREAMING_SNAKE_CASE=7_68      , _SCREAMING_SNAKE_CASE=12      , _SCREAMING_SNAKE_CASE=12      , _SCREAMING_SNAKE_CASE=30_72      , _SCREAMING_SNAKE_CASE="gelu"      , _SCREAMING_SNAKE_CASE=0.1      , _SCREAMING_SNAKE_CASE=0.1      , _SCREAMING_SNAKE_CASE=1_63_84      , _SCREAMING_SNAKE_CASE=16      , _SCREAMING_SNAKE_CASE=0.02      , _SCREAMING_SNAKE_CASE=1e-12      , _SCREAMING_SNAKE_CASE=0      , _SCREAMING_SNAKE_CASE=0Xe0_00      , _SCREAMING_SNAKE_CASE=0Xe0_01      , _SCREAMING_SNAKE_CASE=4      , _SCREAMING_SNAKE_CASE=4      , _SCREAMING_SNAKE_CASE=8      , _SCREAMING_SNAKE_CASE=1_63_84      , _SCREAMING_SNAKE_CASE=1_28      , **_SCREAMING_SNAKE_CASE      , ):





      '''simple docstring'''

      super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE      , bos_token_id=_SCREAMING_SNAKE_CASE      , eos_token_id=_SCREAMING_SNAKE_CASE      , **_SCREAMING_SNAKE_CASE	)

      lowerCAmelCase 						=							max_position_embeddings
      lowerCAmelCase 						=							hidden_size
      lowerCAmelCase 						=							num_hidden_layers
      lowerCAmelCase 						=							num_attention_heads
      lowerCAmelCase 						=							intermediate_size
      lowerCAmelCase 						=							hidden_act
      lowerCAmelCase 						=							hidden_dropout_prob
      lowerCAmelCase 						=							attention_probs_dropout_prob
      lowerCAmelCase 						=							initializer_range
      lowerCAmelCase 						=							type_vocab_size
      lowerCAmelCase 						=							layer_norm_eps

      # Character config:
      lowerCAmelCase 						=							downsampling_rate
      lowerCAmelCase 						=							upsampling_kernel_size
      lowerCAmelCase 						=							num_hash_functions
      lowerCAmelCase 						=							num_hash_buckets
      lowerCAmelCase 						=							local_transformer_stride





