
"""simple docstring"""


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast



class 	__a	(   unittest.TestCase						):






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple      )->	Tuple:







              """simple docstring"""
              # A mock response for an HTTP head request to emulate server down
              UpperCamelCase       			=		mock.Mock()
              UpperCamelCase       			=		500
              UpperCamelCase       			=		{}
              UpperCamelCase       			=		HTTPError
              UpperCamelCase       			=		{}

              # Download this model to make sure it's in the cache.
              UpperCamelCase       			=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"      )

              # Under the mock environment we get a 500 error when trying to reach the tokenizer.
              with mock.patch("requests.Session.request"							,   return_value=UpperCAmelCase_      ) as mock_head:
                     UpperCamelCase       			=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"      )
                     # This check we did call the fake head request
                     mock_head.assert_called()






       @require_tokenizers
       def 						_SCREAMING_SNAKE_CASE							(	self			:				str      )->	Tuple:







              """simple docstring"""
              # A mock response for an HTTP head request to emulate server down
              UpperCamelCase       			=		mock.Mock()
              UpperCamelCase       			=		500
              UpperCamelCase       			=		{}
              UpperCamelCase       			=		HTTPError
              UpperCamelCase       			=		{}

              # Download this model to make sure it's in the cache.
              UpperCamelCase       			=		GPTaTokenizerFast.from_pretrained("gpt2"      )

              # Under the mock environment we get a 500 error when trying to reach the tokenizer.
              with mock.patch("requests.Session.request"							,   return_value=UpperCAmelCase_      ) as mock_head:
                     UpperCamelCase       			=		GPTaTokenizerFast.from_pretrained("gpt2"      )
                     # This check we did call the fake head request
                     mock_head.assert_called()






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]      )->	str:







              """simple docstring"""
              # This test is for deprecated behavior and can be removed in v5
              try:
                     UpperCamelCase       			=		tempfile.mktemp()
                     with open(UpperCAmelCase_							,   "wb"      ) as f:
                            http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"							,   UpperCAmelCase_      )

                     UpperCamelCase       			=		AlbertTokenizer.from_pretrained(UpperCAmelCase_      )
              finally:
                     os.remove(UpperCAmelCase_      )

              # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
              # the current folder and have the right name.
              if os.path.isfile("tokenizer.json"      ):
                     # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                     return
              try:
                     with open("tokenizer.json"							,   "wb"      ) as f:
                            http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"							,   UpperCAmelCase_      )
                     UpperCamelCase       			=		AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"      )
                     # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                     self.assertEqual(tokenizer.vocab_size							,   1_000      )
                     # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

              finally:
                     os.remove("tokenizer.json"      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	Any:







              """simple docstring"""
              # This test is for deprecated behavior and can be removed in v5
              UpperCamelCase       			=		AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"      )





@is_staging_test
class 	__a	(   unittest.TestCase						):
       UpperCamelCase_		:       str				=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






       @classmethod
       def 						_SCREAMING_SNAKE_CASE							(	cls			:				List[str]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		TOKEN
              HfFolder.save_token(UpperCAmelCase_      )






       @classmethod
       def 						_SCREAMING_SNAKE_CASE							(	cls			:				Tuple      )->	Any:







              """simple docstring"""
              try:
                     delete_repo(token=cls._token							,   repo_id="test-tokenizer"      )
              except HTTPError:
                     pass

              try:
                     delete_repo(token=cls._token							,   repo_id="valid_org/test-tokenizer-org"      )
              except HTTPError:
                     pass

              try:
                     delete_repo(token=cls._token							,   repo_id="test-dynamic-tokenizer"      )
              except HTTPError:
                     pass






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]      )->	Optional[Any]:







              """simple docstring"""
              with tempfile.TemporaryDirectory() as tmp_dir:
                     UpperCamelCase       			=		os.path.join(UpperCAmelCase_							,   "vocab.txt"      )
                     with open(UpperCAmelCase_							,   "w"							,   encoding="utf-8"      ) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
                     UpperCamelCase       			=		BertTokenizer(UpperCAmelCase_      )

              tokenizer.push_to_hub("test-tokenizer"							,   use_auth_token=self._token      )
              UpperCamelCase       			=		BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"      )
              self.assertDictEqual(new_tokenizer.vocab							,   tokenizer.vocab      )

              # Reset repo
              delete_repo(token=self._token							,   repo_id="test-tokenizer"      )

              # Push to hub via save_pretrained
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(UpperCAmelCase_							,   repo_id="test-tokenizer"							,   push_to_hub=UpperCAmelCase_							,   use_auth_token=self._token      )

              UpperCamelCase       			=		BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"      )
              self.assertDictEqual(new_tokenizer.vocab							,   tokenizer.vocab      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	Tuple:







              """simple docstring"""
              with tempfile.TemporaryDirectory() as tmp_dir:
                     UpperCamelCase       			=		os.path.join(UpperCAmelCase_							,   "vocab.txt"      )
                     with open(UpperCAmelCase_							,   "w"							,   encoding="utf-8"      ) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
                     UpperCamelCase       			=		BertTokenizer(UpperCAmelCase_      )

              tokenizer.push_to_hub("valid_org/test-tokenizer-org"							,   use_auth_token=self._token      )
              UpperCamelCase       			=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"      )
              self.assertDictEqual(new_tokenizer.vocab							,   tokenizer.vocab      )

              # Reset repo
              delete_repo(token=self._token							,   repo_id="valid_org/test-tokenizer-org"      )

              # Push to hub via save_pretrained
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(
                         UpperCAmelCase_							,   repo_id="valid_org/test-tokenizer-org"							,   push_to_hub=UpperCAmelCase_							,   use_auth_token=self._token      )

              UpperCamelCase       			=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"      )
              self.assertDictEqual(new_tokenizer.vocab							,   tokenizer.vocab      )






       @require_tokenizers
       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	List[Any]:







              """simple docstring"""
              CustomTokenizer.register_for_auto_class()
              with tempfile.TemporaryDirectory() as tmp_dir:
                     UpperCamelCase       			=		os.path.join(UpperCAmelCase_							,   "vocab.txt"      )
                     with open(UpperCAmelCase_							,   "w"							,   encoding="utf-8"      ) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
                     UpperCamelCase       			=		CustomTokenizer(UpperCAmelCase_      )

              # No fast custom tokenizer
              tokenizer.push_to_hub("test-dynamic-tokenizer"							,   use_auth_token=self._token      )

              UpperCamelCase       			=		AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"							,   trust_remote_code=UpperCAmelCase_      )
              # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__							,   "CustomTokenizer"      )

              # Fast and slow custom tokenizer
              CustomTokenizerFast.register_for_auto_class()
              with tempfile.TemporaryDirectory() as tmp_dir:
                     UpperCamelCase       			=		os.path.join(UpperCAmelCase_							,   "vocab.txt"      )
                     with open(UpperCAmelCase_							,   "w"							,   encoding="utf-8"      ) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )

                     UpperCamelCase       			=		BertTokenizerFast.from_pretrained(UpperCAmelCase_      )
                     bert_tokenizer.save_pretrained(UpperCAmelCase_      )
                     UpperCamelCase       			=		CustomTokenizerFast.from_pretrained(UpperCAmelCase_      )

              tokenizer.push_to_hub("test-dynamic-tokenizer"							,   use_auth_token=self._token      )

              UpperCamelCase       			=		AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"							,   trust_remote_code=UpperCAmelCase_      )
              # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__							,   "CustomTokenizerFast"      )
              UpperCamelCase       			=		AutoTokenizer.from_pretrained(
                  f"{USER}/test-dynamic-tokenizer"							,   use_fast=UpperCAmelCase_							,   trust_remote_code=UpperCAmelCase_      )
              # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__							,   "CustomTokenizer"      )





class 	__a	(   unittest.TestCase						):






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	str:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              trie.add("Hello 友達"      )
              self.assertEqual(trie.data							,   {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}      )
              trie.add("Hello"      )
              trie.data
              self.assertEqual(trie.data							,   {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple      )->	Optional[int]:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              self.assertEqual(trie.split("[CLS] This is a extra_id_100"      )							,   ["[CLS] This is a extra_id_100"]      )
              trie.add("[CLS]"      )
              trie.add("extra_id_1"      )
              trie.add("extra_id_100"      )
              self.assertEqual(trie.split("[CLS] This is a extra_id_100"      )							,   ["[CLS]", " This is a ", "extra_id_100"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				int      )->	Dict:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              trie.add("A"      )
              self.assertEqual(trie.split("ABC"      )							,   ["A", "BC"]      )
              self.assertEqual(trie.split("BCA"      )							,   ["BC", "A"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              trie.add("TOKEN]"      )
              trie.add("[SPECIAL_TOKEN]"      )
              self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"      )							,   ["This is something ", "[SPECIAL_TOKEN]"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str      )->	Dict:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              trie.add("A"      )
              trie.add("P"      )
              trie.add("[SPECIAL_TOKEN]"      )
              self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"      )							,   ["This is something ", "[SPECIAL_TOKEN]"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              trie.add("AB"      )
              trie.add("B"      )
              trie.add("C"      )
              self.assertEqual(trie.split("ABC"      )							,   ["AB", "C"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		Trie()
              trie.add("ABC"      )
              trie.add("B"      )
              trie.add("CD"      )
              self.assertEqual(trie.split("ABCD"      )							,   ["ABC", "D"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple      )->	Dict:







              """simple docstring"""
              # Even if the offsets are wrong, we necessarily output correct string
              # parts.
              UpperCamelCase       			=		Trie()
              UpperCamelCase       			=		trie.cut_text("ABC"							,   [0, 0, 2, 1, 2, 3]      )
              self.assertEqual(UpperCAmelCase_							,   ["AB", "C"]      )







"""simple docstring"""


import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class 	__a	(   _lowerCAmelCase     ,  _lowerCAmelCase     ,  _lowerCAmelCase						):






       @register_to_config
       def __init__(	self			:				Union[str, Any]							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				float							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				str							,   UpperCAmelCase_			:				bool = False							,   )->	str:







              """simple docstring"""
              super().__init__()

              UpperCamelCase       			=		nn.Embedding(UpperCAmelCase_							,   UpperCAmelCase_      )

              UpperCamelCase       			=		nn.Embedding(UpperCAmelCase_							,   UpperCAmelCase_      )
              UpperCamelCase       			=		False

              UpperCamelCase       			=		nn.Dropout(p=UpperCAmelCase_      )

              UpperCamelCase       			=		TaConfig(
                  vocab_size=UpperCAmelCase_							,   d_model=UpperCAmelCase_							,   num_heads=UpperCAmelCase_							,   d_kv=UpperCAmelCase_							,   d_ff=UpperCAmelCase_							,   dropout_rate=UpperCAmelCase_							,   feed_forward_proj=UpperCAmelCase_							,   is_decoder=UpperCAmelCase_							,   is_encoder_decoder=UpperCAmelCase_							,   )

              UpperCamelCase       			=		nn.ModuleList()
              for lyr_num in range(UpperCAmelCase_      ):
                     UpperCamelCase       			=		TaBlock(UpperCAmelCase_      )
                     self.encoders.append(UpperCAmelCase_      )

              UpperCamelCase       			=		TaLayerNorm(UpperCAmelCase_      )
              UpperCamelCase       			=		nn.Dropout(p=UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				int							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				str      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase       			=		self.token_embedder(UpperCAmelCase_      )

              UpperCamelCase       			=		encoder_input_tokens.shape[1]
              UpperCamelCase       			=		torch.arange(UpperCAmelCase_							,   device=encoder_input_tokens.device      )
              x += self.position_encoding(UpperCAmelCase_      )

              UpperCamelCase       			=		self.dropout_pre(UpperCAmelCase_      )

              # inverted the attention mask
              UpperCamelCase       			=		encoder_input_tokens.size()
              UpperCamelCase       			=		self.get_extended_attention_mask(UpperCAmelCase_							,   UpperCAmelCase_      )

              for lyr in self.encoders:
                     UpperCamelCase       			=		lyr(UpperCAmelCase_							,   UpperCAmelCase_      )[0]
              UpperCamelCase       			=		self.layer_norm(UpperCAmelCase_      )

              return self.dropout_post(UpperCAmelCase_      ), encoder_inputs_mask





