

"""simple docstring"""
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast

class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Dict							=    mock.Mock()
        UpperCAmelCase__			:							Union[str, Any]							=    500
        UpperCAmelCase__			:							List[str]							=    {}
        UpperCAmelCase__			:							Optional[Any]							=    HTTPError
        UpperCAmelCase__			:							Tuple							=    {}

        # Download this model to make sure it's in the cache.
        UpperCAmelCase__			:							int							=    BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch("""requests.Session.request"""							,       return_value=_lowerCamelCase	) as mock_head:
              UpperCAmelCase__			:							int							=    BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
              # This check we did call the fake head request
              mock_head.assert_called()



  @require_tokenizers
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    mock.Mock()
        UpperCAmelCase__			:							Optional[int]							=    500
        UpperCAmelCase__			:							Any							=    {}
        UpperCAmelCase__			:							Any							=    HTTPError
        UpperCAmelCase__			:							str							=    {}

        # Download this model to make sure it's in the cache.
        UpperCAmelCase__			:							int							=    GPTaTokenizerFast.from_pretrained("""gpt2"""	)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch("""requests.Session.request"""							,       return_value=_lowerCamelCase	) as mock_head:
              UpperCAmelCase__			:							str							=    GPTaTokenizerFast.from_pretrained("""gpt2"""	)
              # This check we did call the fake head request
              mock_head.assert_called()



  def  _a      (self	):


        """simple docstring"""

        try:
              UpperCAmelCase__			:							Any							=    tempfile.mktemp()
              with open(_lowerCamelCase							,       """wb"""	) as f:
                    http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""							,       _lowerCamelCase	)

              UpperCAmelCase__			:							Optional[Any]							=    AlbertTokenizer.from_pretrained(_lowerCamelCase	)
        finally:
              os.remove(_lowerCamelCase	)

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile("""tokenizer.json"""	):
              # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
              return
        try:
              with open("""tokenizer.json"""							,       """wb"""	) as f:
                    http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""							,       _lowerCamelCase	)
              UpperCAmelCase__			:							List[Any]							=    AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""	)
              # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
              self.assertEqual(tokenizer.vocab_size							,       1000	)
              # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
              os.remove("""tokenizer.json"""	)




  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""	)




@is_staging_test
class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']



  @classmethod
  def  _a      (cls	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    TOKEN
        HfFolder.save_token(_lowerCamelCase	)



  @classmethod
  def  _a      (cls	):


        """simple docstring"""

        try:
              delete_repo(token=cls._token							,       repo_id="""test-tokenizer"""	)
        except HTTPError:
              pass

        try:
              delete_repo(token=cls._token							,       repo_id="""valid_org/test-tokenizer-org"""	)
        except HTTPError:
              pass

        try:
              delete_repo(token=cls._token							,       repo_id="""test-dynamic-tokenizer"""	)
        except HTTPError:
              pass



  def  _a      (self	):


        """simple docstring"""

        with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCAmelCase__			:							Optional[int]							=    os.path.join(_lowerCamelCase							,       """vocab.txt"""	)
              with open(_lowerCamelCase							,       """w"""							,       encoding="""utf-8"""	) as vocab_writer:
                    vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
              UpperCAmelCase__			:							Optional[int]							=    BertTokenizer(_lowerCamelCase	)

        tokenizer.push_to_hub("""test-tokenizer"""							,       use_auth_token=self._token	)
        UpperCAmelCase__			:							List[Any]							=    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""	)
        self.assertDictEqual(new_tokenizer.vocab							,       tokenizer.vocab	)

        # Reset repo
        delete_repo(token=self._token							,       repo_id="""test-tokenizer"""	)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(_lowerCamelCase							,       repo_id="""test-tokenizer"""							,       push_to_hub=_lowerCamelCase							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							List[str]							=    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""	)
        self.assertDictEqual(new_tokenizer.vocab							,       tokenizer.vocab	)



  def  _a      (self	):


        """simple docstring"""

        with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCAmelCase__			:							int							=    os.path.join(_lowerCamelCase							,       """vocab.txt"""	)
              with open(_lowerCamelCase							,       """w"""							,       encoding="""utf-8"""	) as vocab_writer:
                    vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
              UpperCAmelCase__			:							Optional[int]							=    BertTokenizer(_lowerCamelCase	)

        tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""							,       use_auth_token=self._token	)
        UpperCAmelCase__			:							Any							=    BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""	)
        self.assertDictEqual(new_tokenizer.vocab							,       tokenizer.vocab	)

        # Reset repo
        delete_repo(token=self._token							,       repo_id="""valid_org/test-tokenizer-org"""	)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(
                  _lowerCamelCase							,       repo_id="""valid_org/test-tokenizer-org"""							,       push_to_hub=_lowerCamelCase							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							int							=    BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""	)
        self.assertDictEqual(new_tokenizer.vocab							,       tokenizer.vocab	)




  @require_tokenizers
  def  _a      (self	):


        """simple docstring"""

        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCAmelCase__			:							int							=    os.path.join(_lowerCamelCase							,       """vocab.txt"""	)
              with open(_lowerCamelCase							,       """w"""							,       encoding="""utf-8"""	) as vocab_writer:
                    vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
              UpperCAmelCase__			:							Optional[Any]							=    CustomTokenizer(_lowerCamelCase	)

        # No fast custom tokenizer
        tokenizer.push_to_hub("""test-dynamic-tokenizer"""							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							List[Any]							=    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""							,       trust_remote_code=_lowerCamelCase	)
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__							,       """CustomTokenizer"""	)

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCAmelCase__			:							int							=    os.path.join(_lowerCamelCase							,       """vocab.txt"""	)
              with open(_lowerCamelCase							,       """w"""							,       encoding="""utf-8"""	) as vocab_writer:
                    vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)

              UpperCAmelCase__			:							Union[str, Any]							=    BertTokenizerFast.from_pretrained(_lowerCamelCase	)
              bert_tokenizer.save_pretrained(_lowerCamelCase	)
              UpperCAmelCase__			:							Dict							=    CustomTokenizerFast.from_pretrained(_lowerCamelCase	)

        tokenizer.push_to_hub("""test-dynamic-tokenizer"""							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							Union[str, Any]							=    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""							,       trust_remote_code=_lowerCamelCase	)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__							,       """CustomTokenizerFast"""	)
        UpperCAmelCase__			:							Tuple							=    AutoTokenizer.from_pretrained(
            F"""{USER}/test-dynamic-tokenizer"""							,       use_fast=_lowerCamelCase							,       trust_remote_code=_lowerCamelCase	)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__							,       """CustomTokenizer"""	)




class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    Trie()
        trie.add("""Hello 友達"""	)
        self.assertEqual(trie.data							,       {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}	)
        trie.add("""Hello"""	)
        trie.data
        self.assertEqual(trie.data							,       {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    Trie()
        self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""	)							,       ["""[CLS] This is a extra_id_100"""]	)
        trie.add("""[CLS]"""	)
        trie.add("""extra_id_1"""	)
        trie.add("""extra_id_100"""	)
        self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""	)							,       ["""[CLS]""", """ This is a """, """extra_id_100"""]	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    Trie()
        trie.add("""A"""	)
        self.assertEqual(trie.split("""ABC"""	)							,       ["""A""", """BC"""]	)
        self.assertEqual(trie.split("""BCA"""	)							,       ["""BC""", """A"""]	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    Trie()
        trie.add("""TOKEN]"""	)
        trie.add("""[SPECIAL_TOKEN]"""	)
        self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""	)							,       ["""This is something """, """[SPECIAL_TOKEN]"""]	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    Trie()
        trie.add("""A"""	)
        trie.add("""P"""	)
        trie.add("""[SPECIAL_TOKEN]"""	)
        self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""	)							,       ["""This is something """, """[SPECIAL_TOKEN]"""]	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    Trie()
        trie.add("""AB"""	)
        trie.add("""B"""	)
        trie.add("""C"""	)
        self.assertEqual(trie.split("""ABC"""	)							,       ["""AB""", """C"""]	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    Trie()
        trie.add("""ABC"""	)
        trie.add("""B"""	)
        trie.add("""CD"""	)
        self.assertEqual(trie.split("""ABCD"""	)							,       ["""ABC""", """D"""]	)




  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    Trie()
        UpperCAmelCase__			:							List[str]							=    trie.cut_text("""ABC"""							,       [0, 0, 2, 1, 2, 3]	)
        self.assertEqual(_lowerCamelCase							,       ["""AB""", """C"""]	)




"""simple docstring"""
def 					a__						(						)	-> Union[str, Any]:
      UpperCAmelCase__			:							Dict							=    []
      UpperCAmelCase__			:							Tuple							=    1

      while len(lowerCAmelCase    ) < 1E6:
            constant.append(str(lowerCAmelCase    )    )
            i += 1

      UpperCAmelCase__			:							Any							=    """""".join(lowerCAmelCase    )

      return (
          int(constant[0]    )
          * int(constant[9]    )
          * int(constant[99]    )
          * int(constant[9_99]    )
          * int(constant[99_99]    )
          * int(constant[9_99_99]    )
          * int(constant[99_99_99]    )
      )


if __name__ == "__main__":
    print(solution())

