






import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 							a_ (						a_	,	unittest.TestCase		):


							'''simple docstring'''






							__a:	Any     =    RealmTokenizer
							__a:	Optional[Any]     =    RealmTokenizerFast
							__a:	List[str]     =    True
							__a:	Tuple     =    True
							__a:	Optional[Any]     =    filter_non_english

							def      _lowercase       (		self   )					->					List[str]:
														'''simple docstring'''

														super().setUp()

														lowerCAmelCase_										=     [
														    '[UNK]',
														    '[CLS]',
														    '[SEP]',
														    '[PAD]',
														    '[MASK]',
														    'want',
														    '##want',
														    '##ed',
														    'wa',
														    'un',
														    'runn',
														    '##ing',
														    ',',
														    'low',
														    'lowest',
														]
														lowerCAmelCase_										=     os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['vocab_file']   )
														with open(self.vocab_file			,			'w'			,			encoding='utf-8'   ) as vocab_writer:
																					vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )

							def      _lowercase       (		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     'UNwant\u00E9d,running'
														lowerCAmelCase_										=     'unwanted, running'
														return input_text, output_text

							def      _lowercase       (		self   )					->					Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     self.tokenizer_class(self.vocab_file   )

														lowerCAmelCase_										=     tokenizer.tokenize('UNwant\u00E9d,running'   )
														self.assertListEqual(lowercase_			,			['un', '##want', '##ed', ',', 'runn', '##ing']   )
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_   )			,			[9, 6, 7, 1_2, 1_0, 1_1]   )

							def      _lowercase       (		self   )					->					int:
														'''simple docstring'''

														if not self.test_rust_tokenizer:
																					return

														lowerCAmelCase_										=     self.get_tokenizer()
														lowerCAmelCase_										=     self.get_rust_tokenizer()

														lowerCAmelCase_										=     'UNwant\u00E9d,running'

														lowerCAmelCase_										=     tokenizer.tokenize(lowercase_   )
														lowerCAmelCase_										=     rust_tokenizer.tokenize(lowercase_   )
														self.assertListEqual(lowercase_			,			lowercase_   )

														lowerCAmelCase_										=     tokenizer.encode(lowercase_			,			add_special_tokens=lowercase_   )
														lowerCAmelCase_										=     rust_tokenizer.encode(lowercase_			,			add_special_tokens=lowercase_   )
														self.assertListEqual(lowercase_			,			lowercase_   )

														lowerCAmelCase_										=     self.get_rust_tokenizer()
														lowerCAmelCase_										=     tokenizer.encode(lowercase_   )
														lowerCAmelCase_										=     rust_tokenizer.encode(lowercase_   )
														self.assertListEqual(lowercase_			,			lowercase_   )

														# With lower casing
														lowerCAmelCase_										=     self.get_tokenizer(do_lower_case=lowercase_   )
														lowerCAmelCase_										=     self.get_rust_tokenizer(do_lower_case=lowercase_   )

														lowerCAmelCase_										=     'UNwant\u00E9d,running'

														lowerCAmelCase_										=     tokenizer.tokenize(lowercase_   )
														lowerCAmelCase_										=     rust_tokenizer.tokenize(lowercase_   )
														self.assertListEqual(lowercase_			,			lowercase_   )

														lowerCAmelCase_										=     tokenizer.encode(lowercase_			,			add_special_tokens=lowercase_   )
														lowerCAmelCase_										=     rust_tokenizer.encode(lowercase_			,			add_special_tokens=lowercase_   )
														self.assertListEqual(lowercase_			,			lowercase_   )

														lowerCAmelCase_										=     self.get_rust_tokenizer()
														lowerCAmelCase_										=     tokenizer.encode(lowercase_   )
														lowerCAmelCase_										=     rust_tokenizer.encode(lowercase_   )
														self.assertListEqual(lowercase_			,			lowercase_   )

							def      _lowercase       (		self   )					->					Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer()

														self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'   )			,			['ah', '\u535A', '\u63A8', 'zz']   )

							def      _lowercase       (		self   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )			,			['hello', '!', 'how', 'are', 'you', '?']   )
														self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )			,			['hello']   )

							def      _lowercase       (		self   )					->					Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_			,			strip_accents=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )			,			['hällo', '!', 'how', 'are', 'you', '?']   )
														self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )			,			['h\u00E9llo']   )

							def      _lowercase       (		self   )					->					List[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_			,			strip_accents=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )			,			['hallo', '!', 'how', 'are', 'you', '?']   )
														self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )			,			['hello']   )

							def      _lowercase       (		self   )					->					str:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )			,			['hallo', '!', 'how', 'are', 'you', '?']   )
														self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )			,			['hello']   )

							def      _lowercase       (		self   )					->					Optional[int]:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )			,			['HeLLo', '!', 'how', 'Are', 'yoU', '?']   )

							def      _lowercase       (		self   )					->					Tuple:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_			,			strip_accents=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )			,			['HäLLo', '!', 'how', 'Are', 'yoU', '?']   )

							def      _lowercase       (		self   )					->					Union[str, Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_			,			strip_accents=lowercase_   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )			,			['HaLLo', '!', 'how', 'Are', 'yoU', '?']   )

							def      _lowercase       (		self   )					->					str:
														'''simple docstring'''

														lowerCAmelCase_										=     BasicTokenizer(do_lower_case=lowercase_			,			never_split=['[UNK]']   )

														self.assertListEqual(
														    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'   )			,			['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']   )

							def      _lowercase       (		self   )					->					Optional[int]:
														'''simple docstring'''

														lowerCAmelCase_										=     ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

														lowerCAmelCase_										=     {}
														for i, token in enumerate(lowercase_   ):
																					lowerCAmelCase_										=     i
														lowerCAmelCase_										=     WordpieceTokenizer(vocab=lowercase_			,			unk_token='[UNK]'   )

														self.assertListEqual(tokenizer.tokenize(''   )			,			[]   )

														self.assertListEqual(tokenizer.tokenize('unwanted running'   )			,			['un', '##want', '##ed', 'runn', '##ing']   )

														self.assertListEqual(tokenizer.tokenize('unwantedX running'   )			,			['[UNK]', 'runn', '##ing']   )

							def      _lowercase       (		self   )					->					List[str]:
														'''simple docstring'''

														self.assertTrue(_is_whitespace(' '   )   )
														self.assertTrue(_is_whitespace('\t'   )   )
														self.assertTrue(_is_whitespace('\r'   )   )
														self.assertTrue(_is_whitespace('\n'   )   )
														self.assertTrue(_is_whitespace('\u00A0'   )   )

														self.assertFalse(_is_whitespace('A'   )   )
														self.assertFalse(_is_whitespace('-'   )   )

							def      _lowercase       (		self   )					->					Optional[Any]:
														'''simple docstring'''

														self.assertTrue(_is_control('\u0005'   )   )

														self.assertFalse(_is_control('A'   )   )
														self.assertFalse(_is_control(' '   )   )
														self.assertFalse(_is_control('\t'   )   )
														self.assertFalse(_is_control('\r'   )   )

							def      _lowercase       (		self   )					->					List[str]:
														'''simple docstring'''

														self.assertTrue(_is_punctuation('-'   )   )
														self.assertTrue(_is_punctuation('$'   )   )
														self.assertTrue(_is_punctuation('`'   )   )
														self.assertTrue(_is_punctuation('.'   )   )

														self.assertFalse(_is_punctuation('A'   )   )
														self.assertFalse(_is_punctuation(' '   )   )

							def      _lowercase       (		self   )					->					List[str]:
														'''simple docstring'''

														lowerCAmelCase_										=     self.get_tokenizer()

														# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
														self.assertListEqual([tokenizer.tokenize(lowercase_   ) for t in ['Test', '\xad', 'test']]			,			[['[UNK]'], [], ['[UNK]']]   )

														if self.test_rust_tokenizer:
																					lowerCAmelCase_										=     self.get_rust_tokenizer()
																					self.assertListEqual(
																					    [rust_tokenizer.tokenize(lowercase_   ) for t in ['Test', '\xad', 'test']]			,			[['[UNK]'], [], ['[UNK]']]   )

							@slow
							def      _lowercase       (		self   )					->					Optional[int]:
														'''simple docstring'''

														lowerCAmelCase_										=     self.tokenizer_class.from_pretrained('bert-base-uncased'   )

														lowerCAmelCase_										=     tokenizer.encode('sequence builders'			,			add_special_tokens=lowercase_   )
														lowerCAmelCase_										=     tokenizer.encode('multi-sequence build'			,			add_special_tokens=lowercase_   )

														lowerCAmelCase_										=     tokenizer.build_inputs_with_special_tokens(lowercase_   )
														lowerCAmelCase_										=     tokenizer.build_inputs_with_special_tokens(lowercase_			,			lowercase_   )

														assert encoded_sentence == [1_0_1] + text + [1_0_2]
														assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

							def      _lowercase       (		self   )					->					Optional[int]:
														'''simple docstring'''

														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																												lowerCAmelCase_										=     self.rust_tokenizer_class.from_pretrained(lowercase_			,			**lowercase_   )

																												lowerCAmelCase_										=     f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																												lowerCAmelCase_										=     tokenizer_r.encode_plus(
																												    lowercase_			,			return_attention_mask=lowercase_			,			return_token_type_ids=lowercase_			,			return_offsets_mapping=lowercase_			,			add_special_tokens=lowercase_			,			)

																												lowerCAmelCase_										=     tokenizer_r.do_lower_case if hasattr(lowercase_			,			'do_lower_case'   ) else False
																												lowerCAmelCase_										=     (
																												    [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), 'A'),
																												        ((1, 2), ','),
																												        ((3, 5), 'na'),
																												        ((5, 6), '##ï'),
																												        ((6, 8), '##ve'),
																												        ((9, 1_5), tokenizer_r.mask_token),
																												        ((1_6, 2_1), 'Allen'),
																												        ((2_1, 2_3), '##NL'),
																												        ((2_3, 2_4), '##P'),
																												        ((2_5, 3_3), 'sentence'),
																												        ((3_3, 3_4), '.'),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												    if not do_lower_case
																												    else [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), 'a'),
																												        ((1, 2), ','),
																												        ((3, 8), 'naive'),
																												        ((9, 1_5), tokenizer_r.mask_token),
																												        ((1_6, 2_1), 'allen'),
																												        ((2_1, 2_3), '##nl'),
																												        ((2_3, 2_4), '##p'),
																												        ((2_5, 3_3), 'sentence'),
																												        ((3_3, 3_4), '.'),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												)

																												self.assertEqual(
																												    [e[1] for e in expected_results]			,			tokenizer_r.convert_ids_to_tokens(tokens['input_ids']   )   )
																												self.assertEqual([e[0] for e in expected_results]			,			tokens['offset_mapping']   )






							@slow
							def      _lowercase       (		self   )					->					Tuple:
														'''simple docstring'''

														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																												lowerCAmelCase_										=     self.rust_tokenizer_class.from_pretrained(lowercase_			,			**lowercase_   )
																												lowerCAmelCase_										=     self.tokenizer_class.from_pretrained(lowercase_			,			**lowercase_   )
																												lowerCAmelCase_										=     [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																												lowerCAmelCase_										=     tokenizer_r.batch_encode_candidates(lowercase_			,			max_length=1_0			,			return_tensors='np'   )
																												lowerCAmelCase_										=     tokenizer_p.batch_encode_candidates(lowercase_			,			max_length=1_0			,			return_tensors='np'   )

																												lowerCAmelCase_										=     (2, 2, 1_0)

																												self.assertEqual(encoded_sentence_r['input_ids'].shape			,			lowercase_   )
																												self.assertEqual(encoded_sentence_r['attention_mask'].shape			,			lowercase_   )
																												self.assertEqual(encoded_sentence_r['token_type_ids'].shape			,			lowercase_   )

																												self.assertEqual(encoded_sentence_p['input_ids'].shape			,			lowercase_   )
																												self.assertEqual(encoded_sentence_p['attention_mask'].shape			,			lowercase_   )
																												self.assertEqual(encoded_sentence_p['token_type_ids'].shape			,			lowercase_   )











import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


lowerCamelCase_						     =  logging.get_logger(__name__)




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					Any:
														'''simple docstring'''

														super().__init__()
														if version.parse(torch.__version__   ) < version.parse('1.12.0'   ):
																					raise ImportError(
																					    f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
																					    'PytorchGELUTanh. Please upgrade torch.'   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return nn.functional.gelu(lowercase_			,			approximate='tanh'   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi   ) * (input + 0.04_47_15 * torch.pow(lowercase_			,			3.0   ))   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self			,			lowercase_ = False   )					->					Optional[Any]:
														'''simple docstring'''

														super().__init__()
														if use_gelu_python:
																					lowerCAmelCase_										=     self._gelu_python
														else:
																					lowerCAmelCase_										=     nn.functional.gelu

							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0   )   ))






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return self.act(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1.0 + torch.tanh(input * 0.79_78_84_56_08 * (1.0 + 0.04_47_15 * input * input)   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * torch.sigmoid(1.7_02 * input   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self			,			lowercase_			,			lowercase_   )					->					Optional[int]:
														'''simple docstring'''

														if min > max:
																					raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''   )

														super().__init__()
														lowerCAmelCase_										=     min
														lowerCAmelCase_										=     max






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return torch.clip(gelu(lowercase_   )			,			self.min			,			self.max   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					Union[str, Any]:
														'''simple docstring'''

														super().__init__()
														lowerCAmelCase_										=     math.sqrt(2 / math.pi   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_47_15 * torch.pow(lowercase_			,			3   ))   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return nn.functional.silu(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					List[Any]:
														'''simple docstring'''

														super().__init__()
														if version.parse(torch.__version__   ) < version.parse('1.9.0'   ):
																					lowerCAmelCase_										=     self._mish_python
														else:
																					lowerCAmelCase_										=     nn.functional.mish

							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * torch.tanh(nn.functional.softplus(lowercase_   )   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return self.act(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_			,			lowercase_=0.70_71_07			,			lowercase_=0.28_20_95   )					->					Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     (input - mu).div(sigma * math.sqrt(2.0   )   )
														return 0.5 * (1.0 + torch.erf(lowercase_   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     nn.functional.relu(lowercase_   )
														lowerCAmelCase_										=     torch.square(lowercase_   )
														return squared




class 							a_ (						a_		):


							'''simple docstring'''






							def __getitem__(		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     super().__getitem__(lowercase_   )
														lowerCAmelCase_						,   lowerCAmelCase_										=     content if isinstance(lowercase_			,			lowercase_   ) else (content, {})
														return cls(**lowercase_   )


lowerCamelCase_						     =  {
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -1_0, """max""": 1_0}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
lowerCamelCase_						     =  ClassInstantier(ACTaCLS)
def      lowerCamelCase					(						a_	)	->			str:
							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()	)}'''	)


# For backwards compatibility with: from activations import gelu_python
lowerCamelCase_						     =  get_activation("""gelu_python""")
lowerCamelCase_						     =  get_activation("""gelu_new""")
lowerCamelCase_						     =  get_activation("""gelu""")
lowerCamelCase_						     =  get_activation("""gelu_fast""")
lowerCamelCase_						     =  get_activation("""quick_gelu""")
lowerCamelCase_						     =  get_activation("""silu""")
lowerCamelCase_						     =  get_activation("""mish""")



lowerCamelCase_						     =  get_activation("""linear""")



