




import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					__lowerCamelCase			(       UpperCamelCase__      ,							unittest.TestCase       ):




    """simple docstring"""

    snake_case__   = OpenAIGPTTokenizer
    snake_case__   = OpenAIGPTTokenizerFast
    snake_case__   = True
    snake_case__   = False


    def   a (			self	:	Dict  ) ->			str:
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         lowerCAmelCase__            =					[
             "l",
             "o",
             "w",
             "e",
             "r",
             "s",
             "t",
             "i",
             "d",
             "n",
             "w</w>",
             "r</w>",
             "t</w>",
             "lo",
             "low",
             "er</w>",
             "low</w>",
             "lowest</w>",
             "newer</w>",
             "wider</w>",
             "<unk>",
         ]
         lowerCAmelCase__            =					dict(zip(SCREAMING_SNAKE_CASE__  ,      range(len(SCREAMING_SNAKE_CASE__  )  )  )  )
         lowerCAmelCase__            =					["#version: 0.2", "l o", "lo w", "e r</w>", ""]

         lowerCAmelCase__            =					os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["vocab_file"]  )
         lowerCAmelCase__            =					os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["merges_file"]  )
         with open(self.vocab_file  ,      "w"  ) as fp:
              fp.write(json.dumps(SCREAMING_SNAKE_CASE__  )  )
         with open(self.merges_file  ,      "w"  ) as fp:
              fp.write("\n".join(SCREAMING_SNAKE_CASE__  )  )


    def   a (			self	:	Dict  ,      SCREAMING_SNAKE_CASE__	:	Optional[Any]  ) ->			Optional[Any]:
         return "lower newer", "lower newer"


    def   a (			self	:	List[str]  ) ->			Dict:
         lowerCAmelCase__            =					OpenAIGPTTokenizer(self.vocab_file  ,      self.merges_file  )

         lowerCAmelCase__            =					"lower"
         lowerCAmelCase__            =					["low", "er</w>"]
         lowerCAmelCase__            =					tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokens + ["<unk>"]
         lowerCAmelCase__            =					[14, 15, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )  ,      SCREAMING_SNAKE_CASE__  )


    def   a (			self	:	List[str]  ,      SCREAMING_SNAKE_CASE__	:	Any=15  ) ->			Tuple:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                   lowerCAmelCase__            =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )

                   # Simple input
                   lowerCAmelCase__            =					"This is a simple input"
                   lowerCAmelCase__            =					["This is a simple input 1", "This is a simple input 2"]
                   lowerCAmelCase__            =					("This is a simple input", "This is a pair")
                   lowerCAmelCase__            =					[
                       ("This is a simple input 1", "This is a simple input 2"),
                       ("This is a simple pair 1", "This is a simple pair 2"),
                   ]

                   # Simple input tests
                   self.assertRaises(SCREAMING_SNAKE_CASE__  ,      tokenizer_r.encode  ,      SCREAMING_SNAKE_CASE__  ,      max_length=SCREAMING_SNAKE_CASE__  ,      padding="max_length"  )

                   # Simple input
                   self.assertRaises(SCREAMING_SNAKE_CASE__  ,      tokenizer_r.encode_plus  ,      SCREAMING_SNAKE_CASE__  ,      max_length=SCREAMING_SNAKE_CASE__  ,      padding="max_length"  )

                   # Simple input
                   self.assertRaises(
                       SCREAMING_SNAKE_CASE__  ,      tokenizer_r.batch_encode_plus  ,      SCREAMING_SNAKE_CASE__  ,      max_length=SCREAMING_SNAKE_CASE__  ,      padding="max_length"  ,      )

                   # Pair input
                   self.assertRaises(SCREAMING_SNAKE_CASE__  ,      tokenizer_r.encode  ,      SCREAMING_SNAKE_CASE__  ,      max_length=SCREAMING_SNAKE_CASE__  ,      padding="max_length"  )

                   # Pair input
                   self.assertRaises(SCREAMING_SNAKE_CASE__  ,      tokenizer_r.encode_plus  ,      SCREAMING_SNAKE_CASE__  ,      max_length=SCREAMING_SNAKE_CASE__  ,      padding="max_length"  )

                   # Pair input
                   self.assertRaises(
                       SCREAMING_SNAKE_CASE__  ,      tokenizer_r.batch_encode_plus  ,      SCREAMING_SNAKE_CASE__  ,      max_length=SCREAMING_SNAKE_CASE__  ,      padding="max_length"  ,      )


    def   a (			self	:	Optional[int]  ) ->			int:
         pass






@require_ftfy
@require_spacy
@require_tokenizers
class 					__lowerCamelCase			(       UpperCamelCase__       ):




    """simple docstring"""

    pass






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
					import torch

					from transformers.activations import gelu_new, gelu_python, get_activation







@require_torch
class        __lowerCAmelCase					(			unittest.TestCase					):

			"""simple docstring"""







			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]				)
							__lowerCamelCase									=       get_activation('gelu'				)
							self.assertTrue(torch.allclose(gelu_python(lowerCamelCase__				)     ,		torch_builtin(lowerCamelCase__				)				)				)
							self.assertFalse(torch.allclose(gelu_python(lowerCamelCase__				)     ,		gelu_new(lowerCamelCase__				)				)				)


			def  lowercase_       ( self				)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]				)
							__lowerCamelCase									=       get_activation('gelu'				)
							__lowerCamelCase									=       get_activation('gelu_10'				)

							__lowerCamelCase									=       torch_builtin(lowerCamelCase__				)
							__lowerCamelCase									=       geluaa(lowerCamelCase__				)

							__lowerCamelCase									=       torch.where(y_gelu_aa < 10.0     ,		1     ,		0				)

							self.assertTrue(torch.max(lowerCamelCase__				).item() == 10.0				)
							self.assertTrue(torch.allclose(y_gelu * clipped_mask     ,		y_gelu_aa * clipped_mask				)				)


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							get_activation('gelu'				)
							get_activation('gelu_10'				)
							get_activation('gelu_fast'				)
							get_activation('gelu_new'				)
							get_activation('gelu_python'				)
							get_activation('gelu_pytorch_tanh'				)
							get_activation('linear'				)
							get_activation('mish'				)
							get_activation('quick_gelu'				)
							get_activation('relu'				)
							get_activation('sigmoid'				)
							get_activation('silu'				)
							get_activation('swish'				)
							get_activation('tanh'				)
							with self.assertRaises(lowerCamelCase__				):
											get_activation('bogus'				)
							with self.assertRaises(lowerCamelCase__				):
											get_activation(lowerCamelCase__				)






			def  lowercase_       ( self				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       get_activation('gelu'				)
							__lowerCamelCase									=       1
							__lowerCamelCase									=       get_activation('gelu'				)
							self.assertEqual(acta.a     ,		1				)
							with self.assertRaises(lowerCamelCase__				):
											__lowerCamelCase									=       acta.a


