





'''simple docstring'''



from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers.models.layoutlm.modeling_tf_layoutlm import (
     TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFLayoutLMForMaskedLM,
     TFLayoutLMForQuestionAnswering,
     TFLayoutLMForSequenceClassification,
     TFLayoutLMForTokenClassification,
     TFLayoutLMModel,
 )


class       __SCREAMING_SNAKE_CASE	:



     '''simple docstring'''






     def __init__(					self						,   lowerCamelCase						,   lowerCamelCase=13						,   lowerCamelCase=7						,   lowerCamelCase=True						,   lowerCamelCase=True						,   lowerCamelCase=True						,   lowerCamelCase=True						,   lowerCamelCase=99						,   lowerCamelCase=32						,   lowerCamelCase=2						,   lowerCamelCase=4						,   lowerCamelCase=37						,   lowerCamelCase="gelu"						,   lowerCamelCase=0.1						,   lowerCamelCase=0.1						,   lowerCamelCase=512						,   lowerCamelCase=16						,   lowerCamelCase=2						,   lowerCamelCase=0.02						,   lowerCamelCase=3						,   lowerCamelCase=4						,   lowerCamelCase=None						,   lowerCamelCase=1_000						,   ):
       _snake_case     		=				parent
       _snake_case     		=				batch_size
       _snake_case     		=				seq_length
       _snake_case     		=				is_training
       _snake_case     		=				use_input_mask
       _snake_case     		=				use_token_type_ids
       _snake_case     		=				use_labels
       _snake_case     		=				vocab_size
       _snake_case     		=				hidden_size
       _snake_case     		=				num_hidden_layers
       _snake_case     		=				num_attention_heads
       _snake_case     		=				intermediate_size
       _snake_case     		=				hidden_act
       _snake_case     		=				hidden_dropout_prob
       _snake_case     		=				attention_probs_dropout_prob
       _snake_case     		=				max_position_embeddings
       _snake_case     		=				type_vocab_size
       _snake_case     		=				type_sequence_label_size
       _snake_case     		=				initializer_range
       _snake_case     		=				num_labels
       _snake_case     		=				num_choices
       _snake_case     		=				scope
       _snake_case     		=				range_bbox

     def 	UpperCamelCase(					self	):
       _snake_case     		=				ids_tensor([self.batch_size, self.seq_length]						,   self.vocab_size	)

       # convert bbox to numpy since TF does not support item assignment
       _snake_case     		=				ids_tensor([self.batch_size, self.seq_length, 4]						,   self.range_bbox	).numpy()
       # Ensure that bbox is legal
       for i in range(bbox.shape[0]	):
         for j in range(bbox.shape[1]	):
           if bbox[i, j, 3] < bbox[i, j, 1]:
             _snake_case     		=				bbox[i, j, 3]
             _snake_case     		=				bbox[i, j, 1]
             _snake_case     		=				t
           if bbox[i, j, 2] < bbox[i, j, 0]:
             _snake_case     		=				bbox[i, j, 2]
             _snake_case     		=				bbox[i, j, 0]
             _snake_case     		=				t
       _snake_case     		=				tf.convert_to_tensor(lowerCamelCase	)

       _snake_case     		=				None
       if self.use_input_mask:
         _snake_case     		=				random_attention_mask([self.batch_size, self.seq_length]	)

       _snake_case     		=				None
       if self.use_token_type_ids:
         _snake_case     		=				ids_tensor([self.batch_size, self.seq_length]						,   self.type_vocab_size	)

       _snake_case     		=				None
       _snake_case     		=				None
       _snake_case     		=				None
       if self.use_labels:
         _snake_case     		=				ids_tensor([self.batch_size]						,   self.type_sequence_label_size	)
         _snake_case     		=				ids_tensor([self.batch_size, self.seq_length]						,   self.num_labels	)
         _snake_case     		=				ids_tensor([self.batch_size]						,   self.num_choices	)

       _snake_case     		=				LayoutLMConfig(
           vocab_size=self.vocab_size						,   hidden_size=self.hidden_size						,   num_hidden_layers=self.num_hidden_layers						,   num_attention_heads=self.num_attention_heads						,   intermediate_size=self.intermediate_size						,   hidden_act=self.hidden_act						,   hidden_dropout_prob=self.hidden_dropout_prob						,   attention_probs_dropout_prob=self.attention_probs_dropout_prob						,   max_position_embeddings=self.max_position_embeddings						,   type_vocab_size=self.type_vocab_size						,   initializer_range=self.initializer_range						,   )

       return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase	):
       _snake_case     		=				TFLayoutLMModel(config=lowerCamelCase	)

       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase	)
       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase						,   token_type_ids=lowerCamelCase	)
       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase	)
       self.parent.assertEqual(result.last_hidden_state.shape						,   (self.batch_size, self.seq_length, self.hidden_size)	)
       self.parent.assertEqual(result.pooler_output.shape						,   (self.batch_size, self.hidden_size)	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase	):
       _snake_case     		=				TFLayoutLMForMaskedLM(config=lowerCamelCase	)

       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase						,   labels=lowerCamelCase	)
       self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.seq_length, self.vocab_size)	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase	):
       _snake_case     		=				self.num_labels
       _snake_case     		=				TFLayoutLMForSequenceClassification(config=lowerCamelCase	)

       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase	)
       self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.num_labels)	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase	):
       _snake_case     		=				self.num_labels
       _snake_case     		=				TFLayoutLMForTokenClassification(config=lowerCamelCase	)

       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase						,   labels=lowerCamelCase	)
       self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.seq_length, self.num_labels)	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase	):
       _snake_case     		=				TFLayoutLMForQuestionAnswering(config=lowerCamelCase	)

       _snake_case     		=				model(lowerCamelCase						,   lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase	)
       self.parent.assertEqual(result.start_logits.shape						,   (self.batch_size, self.seq_length)	)
       self.parent.assertEqual(result.end_logits.shape						,   (self.batch_size, self.seq_length)	)






     def 	UpperCamelCase(					self	):
       _snake_case     		=				self.prepare_config_and_inputs()
       (
           (
           _snake_case
       )		,		(
           _snake_case
       )		,		(
           _snake_case
       )		,		(
           _snake_case
       )		,		(
           _snake_case
       )		,		(
           _snake_case
       )		,		(
           _snake_case
       )		,		(
           _snake_case
       )		,		
       )     		=				config_and_inputs
       _snake_case     		=				{
           "input_ids": input_ids,
           "bbox": bbox,
           "token_type_ids": token_type_ids,
           "attention_mask": input_mask,
       }
       return config, inputs_dict







@require_tf
class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase					,    __UpperCamelCase					,    unittest.TestCase			):



     '''simple docstring'''






     UpperCAmelCase__       :						Tuple      =   (
         (
             TFLayoutLMModel,
             TFLayoutLMForMaskedLM,
             TFLayoutLMForTokenClassification,
             TFLayoutLMForSequenceClassification,
             TFLayoutLMForQuestionAnswering,
         )
         if is_tf_available()
         else ()
     )
     UpperCAmelCase__       :						Optional[Any]      =   (
         {
             '''feature-extraction''': TFLayoutLMModel,
             '''fill-mask''': TFLayoutLMForMaskedLM,
             '''text-classification''': TFLayoutLMForSequenceClassification,
             '''token-classification''': TFLayoutLMForTokenClassification,
             '''zero-shot''': TFLayoutLMForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     UpperCAmelCase__       :						int      =   False
     UpperCAmelCase__       :						List[str]      =   True
     UpperCAmelCase__       :						List[Any]      =   10

     def 	UpperCamelCase(					self	):
       _snake_case     		=				TFLayoutLMModelTester(self	)
       _snake_case     		=				ConfigTester(self						,   config_class=lowerCamelCase						,   hidden_size=37	)

     def 	UpperCamelCase(					self	):
       self.config_tester.run_common_tests()

     def 	UpperCamelCase(					self	):
       _snake_case     		=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*lowerCamelCase	)

     def 	UpperCamelCase(					self	):
       _snake_case     		=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase	)

     def 	UpperCamelCase(					self	):
       _snake_case     		=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase	)

     def 	UpperCamelCase(					self	):
       _snake_case     		=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*lowerCamelCase	)

     def 	UpperCamelCase(					self	):
       _snake_case     		=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*lowerCamelCase	)

     @slow
     def 	UpperCamelCase(					self	):
       for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _snake_case     		=				TFLayoutLMModel.from_pretrained(lowerCamelCase	)
         self.assertIsNotNone(lowerCamelCase	)






     @unittest.skip("Onnx compliancy broke with TF 2.10"	)
     def 	UpperCamelCase(					self	):
       pass







def 				snake_case_      (		):







  '''simple docstring'''



  _snake_case     		=				tf.convert_to_tensor([[1_01,10_19,10_14,10_16,10_37,1_28_49,47_47,10_04,1_42_46,22_78,54_39,45_24,50_02,29_30,21_93,29_30,43_41,32_08,10_05,10_55,21_71,28_48,1_13_00,35_31,1_02],[1_01,40_70,40_34,70_20,10_24,30_58,10_15,10_13,28_61,10_13,60_70,1_92_74,27_72,62_05,2_78_14,1_61_47,1_61_47,43_43,20_47,1_02_83,1_09_69,1_43_89,10_12,23_38,1_02]]       )  # noqa: E231
  _snake_case     		=				tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]       )  # noqa: E231
  _snake_case     		=				tf.convert_to_tensor([[[0,0,0,0],[4_23,2_37,4_40,2_51],[4_27,2_72,4_41,2_87],[4_19,1_15,4_37,1_29],[9_61,8_85,9_92,9_12],[2_56,38,3_30,58],[2_56,38,3_30,58],[3_36,42,3_53,57],[3_60,39,4_01,56],[3_60,39,4_01,56],[4_11,39,4_71,59],[4_79,41,5_28,59],[5_33,39,6_30,60],[67,1_13,1_34,1_31],[1_41,1_15,2_09,1_32],[68,1_49,1_33,1_66],[1_41,1_49,1_87,1_64],[1_95,1_48,2_87,1_65],[1_95,1_48,2_87,1_65],[1_95,1_48,2_87,1_65],[2_95,1_48,3_49,1_65],[4_41,1_49,4_92,1_66],[4_97,1_49,5_46,1_64],[64,2_01,1_25,2_18],[10_00,10_00,10_00,10_00]],[[0,0,0,0],[6_62,1_50,7_54,1_66],[6_65,1_99,7_42,2_11],[5_19,2_13,5_54,2_28],[5_19,2_13,5_54,2_28],[1_34,4_33,1_87,4_54],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[1_30,4_67,2_04,4_80],[3_14,4_69,3_76,4_82],[5_04,6_84,5_82,7_06],[9_41,8_25,9_73,9_00],[9_41,8_25,9_73,9_00],[9_41,8_25,9_73,9_00],[9_41,8_25,9_73,9_00],[6_10,7_49,6_52,7_65],[1_30,6_59,1_68,6_72],[1_76,6_57,2_37,6_72],[2_38,6_57,3_12,6_72],[4_43,6_53,6_28,6_72],[4_43,6_53,6_28,6_72],[7_16,3_01,8_25,3_17],[10_00,10_00,10_00,10_00]]]       )  # noqa: E231
  _snake_case     		=				tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]       )  # noqa: E231
  # these are sequence labels (i.e. at the token level)
  _snake_case     		=				tf.convert_to_tensor([[-1_00,10,10,10,9,1,-1_00,7,7,-1_00,7,7,4,2,5,2,8,8,-1_00,-1_00,5,0,3,2,-1_00],[-1_00,12,12,12,-1_00,12,10,-1_00,-1_00,-1_00,-1_00,10,12,9,-1_00,-1_00,-1_00,10,10,10,9,12,-1_00,10,-1_00]]       )  # noqa: E231
  # fmt: on

  return input_ids, attention_mask, bbox, token_type_ids, labels




@require_tf
class       __SCREAMING_SNAKE_CASE	(						unittest.TestCase			):



     '''simple docstring'''






     @slow
     def 	UpperCamelCase(					self	):
       _snake_case     		=				TFLayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased"	)

       _snake_case		,		_snake_case		,		_snake_case		,		_snake_case		,		_snake_case     		=				prepare_layoutlm_batch_inputs()

       # forward pass
       _snake_case     		=				model(input_ids=lowerCamelCase						,   bbox=lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase	)

       # test the sequence output on [0, :3, :3]
       _snake_case     		=				tf.convert_to_tensor(
           [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]]						,   )

       self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]						,   lowerCamelCase						,   atol=1e-3	)	)

       # test the pooled output on [1, :3]
       _snake_case     		=				tf.convert_to_tensor([-0.6580, -0.0214, 0.8552]	)

       self.assertTrue(np.allclose(outputs.pooler_output[1, :3]						,   lowerCamelCase						,   atol=1e-3	)	)

     @slow
     def 	UpperCamelCase(					self	):
       # initialize model with randomly initialized sequence classification head
       _snake_case     		=				TFLayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased"						,   num_labels=2	)

       _snake_case		,		_snake_case		,		_snake_case		,		_snake_case		,		_snake_case     		=				prepare_layoutlm_batch_inputs()

       # forward pass
       _snake_case     		=				model(
           input_ids=lowerCamelCase						,   bbox=lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase						,   labels=tf.convert_to_tensor([1, 1]	)						,   )

       # test whether we get a loss as a scalar
       _snake_case     		=				outputs.loss
       _snake_case     		=				(2,)
       self.assertEqual(loss.shape						,   lowerCamelCase	)

       # test the shape of the logits
       _snake_case     		=				outputs.logits
       _snake_case     		=				(2, 2)
       self.assertEqual(logits.shape						,   lowerCamelCase	)

     @slow
     def 	UpperCamelCase(					self	):
       # initialize model with randomly initialized token classification head
       _snake_case     		=				TFLayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased"						,   num_labels=13	)

       _snake_case		,		_snake_case		,		_snake_case		,		_snake_case		,		_snake_case     		=				prepare_layoutlm_batch_inputs()

       # forward pass
       _snake_case     		=				model(
           input_ids=lowerCamelCase						,   bbox=lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase						,   labels=lowerCamelCase	)

       # test the shape of the logits
       _snake_case     		=				outputs.logits
       _snake_case     		=				tf.convert_to_tensor((2, 25, 13)	)
       self.assertEqual(logits.shape						,   lowerCamelCase	)






     @slow
     def 	UpperCamelCase(					self	):
       # initialize model with randomly initialized token classification head
       _snake_case     		=				TFLayoutLMForQuestionAnswering.from_pretrained("microsoft/layoutlm-base-uncased"	)

       _snake_case		,		_snake_case		,		_snake_case		,		_snake_case		,		_snake_case     		=				prepare_layoutlm_batch_inputs()

       # forward pass
       _snake_case     		=				model(input_ids=lowerCamelCase						,   bbox=lowerCamelCase						,   attention_mask=lowerCamelCase						,   token_type_ids=lowerCamelCase	)

       # test the shape of the logits
       _snake_case     		=				tf.convert_to_tensor((2, 25)	)
       self.assertEqual(outputs.start_logits.shape						,   lowerCamelCase	)
       self.assertEqual(outputs.end_logits.shape						,   lowerCamelCase	)









'''simple docstring'''



import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__   :						Optional[Any]    =			get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase					,    unittest.TestCase			):



     '''simple docstring'''






     UpperCAmelCase__       :						Any      =   ReformerTokenizer
     UpperCAmelCase__       :						List[str]      =   ReformerTokenizerFast
     UpperCAmelCase__       :						Any      =   True
     UpperCAmelCase__       :						Union[str, Any]      =   False
     UpperCAmelCase__       :						Optional[Any]      =   True

     def 	UpperCamelCase(					self	):
       super().setUp()

       _snake_case     		=				ReformerTokenizer(lowerCamelCase						,   keep_accents=lowerCamelCase	)
       tokenizer.save_pretrained(self.tmpdirname	)

     def 	UpperCamelCase(					self	):
       _snake_case     		=				"<s>"
       _snake_case     		=				1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase	)						,   lowerCamelCase	)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase	)						,   lowerCamelCase	)

     def 	UpperCamelCase(					self	):
       _snake_case     		=				list(self.get_tokenizer().get_vocab().keys()	)

       self.assertEqual(vocab_keys[0]						,   "<unk>"	)
       self.assertEqual(vocab_keys[1]						,   "<s>"	)
       self.assertEqual(vocab_keys[-1]						,   "j"	)
       self.assertEqual(len(lowerCamelCase	)						,   1_000	)

     def 	UpperCamelCase(					self	):
       self.assertEqual(self.get_tokenizer().vocab_size						,   1_000	)

     def 	UpperCamelCase(					self	):
       if not self.test_rust_tokenizer:
         return

       _snake_case     		=				self.get_tokenizer()
       _snake_case     		=				self.get_rust_tokenizer()

       _snake_case     		=				"I was born in 92000, and this is falsé."

       _snake_case     		=				tokenizer.tokenize(lowerCamelCase	)
       _snake_case     		=				rust_tokenizer.tokenize(lowerCamelCase	)
       self.assertListEqual(lowerCamelCase						,   lowerCamelCase	)

       _snake_case     		=				tokenizer.encode(lowerCamelCase						,   add_special_tokens=lowerCamelCase	)
       _snake_case     		=				rust_tokenizer.encode(lowerCamelCase						,   add_special_tokens=lowerCamelCase	)
       self.assertListEqual(lowerCamelCase						,   lowerCamelCase	)

       _snake_case     		=				self.get_rust_tokenizer()
       _snake_case     		=				tokenizer.encode(lowerCamelCase	)
       _snake_case     		=				rust_tokenizer.encode(lowerCamelCase	)
       self.assertListEqual(lowerCamelCase						,   lowerCamelCase	)

     def 	UpperCamelCase(					self						,   lowerCamelCase=15	):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
           _snake_case     		=				self.rust_tokenizer_class.from_pretrained(lowerCamelCase						,   **lowerCamelCase	)

           # Simple input
           _snake_case     		=				"This is a simple input"
           _snake_case     		=				["This is a simple input 1", "This is a simple input 2"]
           _snake_case     		=				("This is a simple input", "This is a pair")
           _snake_case     		=				[
               ("This is a simple input 1", "This is a simple input 2"),
               ("This is a simple pair 1", "This is a simple pair 2"),
           ]

           # Simple input tests
           self.assertRaises(lowerCamelCase						,   tokenizer_r.encode						,   lowerCamelCase						,   max_length=lowerCamelCase						,   padding="max_length"	)

           # Simple input
           self.assertRaises(lowerCamelCase						,   tokenizer_r.encode_plus						,   lowerCamelCase						,   max_length=lowerCamelCase						,   padding="max_length"	)

           # Simple input
           self.assertRaises(
               lowerCamelCase						,   tokenizer_r.batch_encode_plus						,   lowerCamelCase						,   max_length=lowerCamelCase						,   padding="max_length"						,   )

           # Pair input
           self.assertRaises(lowerCamelCase						,   tokenizer_r.encode						,   lowerCamelCase						,   max_length=lowerCamelCase						,   padding="max_length"	)

           # Pair input
           self.assertRaises(lowerCamelCase						,   tokenizer_r.encode_plus						,   lowerCamelCase						,   max_length=lowerCamelCase						,   padding="max_length"	)

           # Pair input
           self.assertRaises(
               lowerCamelCase						,   tokenizer_r.batch_encode_plus						,   lowerCamelCase						,   max_length=lowerCamelCase						,   padding="max_length"						,   )

     def 	UpperCamelCase(					self	):
       pass

     def 	UpperCamelCase(					self	):
       _snake_case     		=				ReformerTokenizer(lowerCamelCase						,   keep_accents=lowerCamelCase	)

       _snake_case     		=				tokenizer.tokenize("This is a test"	)
       self.assertListEqual(lowerCamelCase						,   ["▁This", "▁is", "▁a", "▁t", "est"]	)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCamelCase	)						,   [285, 46, 10, 170, 382]						,   )

       _snake_case     		=				tokenizer.tokenize("I was born in 92000, and this is falsé."	)
       self.assertListEqual(
           lowerCamelCase						,   [
               SPIECE_UNDERLINE + "I",
               SPIECE_UNDERLINE + "was",
               SPIECE_UNDERLINE + "b",
               "or",
               "n",
               SPIECE_UNDERLINE + "in",
               SPIECE_UNDERLINE + "",
               "9",
               "2",
               "0",
               "0",
               "0",
               ",",
               SPIECE_UNDERLINE + "and",
               SPIECE_UNDERLINE + "this",
               SPIECE_UNDERLINE + "is",
               SPIECE_UNDERLINE + "f",
               "al",
               "s",
               "é",
               ".",
           ]						,   )
       _snake_case     		=				tokenizer.convert_tokens_to_ids(lowerCamelCase	)
       self.assertListEqual(
           lowerCamelCase						,   [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						,   )

       _snake_case     		=				tokenizer.convert_ids_to_tokens(lowerCamelCase	)
       self.assertListEqual(
           lowerCamelCase						,   [
               SPIECE_UNDERLINE + "I",
               SPIECE_UNDERLINE + "was",
               SPIECE_UNDERLINE + "b",
               "or",
               "n",
               SPIECE_UNDERLINE + "in",
               SPIECE_UNDERLINE + "",
               "<unk>",
               "2",
               "0",
               "0",
               "0",
               ",",
               SPIECE_UNDERLINE + "and",
               SPIECE_UNDERLINE + "this",
               SPIECE_UNDERLINE + "is",
               SPIECE_UNDERLINE + "f",
               "al",
               "s",
               "<unk>",
               ".",
           ]						,   )

     @cached_property
     def 	UpperCamelCase(					self	):
       return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"	)

     @slow
     def 	UpperCamelCase(					self	):
       _snake_case     		=				"Hello World!"
       _snake_case     		=				[126, 32, 262, 152, 38, 72, 287]

       self.assertListEqual(lowerCamelCase						,   self.big_tokenizer.encode(lowerCamelCase	)	)

     @slow
     def 	UpperCamelCase(					self	):
       _snake_case     		=				(
           "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
           " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
       )
       _snake_case     		=				[
           108,
           265,
           24,
           111,
           4,
           258,
           156,
           35,
           28,
           275,
           3,
           259,
           297,
           260,
           84,
           4,
           35,
           110,
           44,
           8,
           259,
           91,
           268,
           21,
           11,
           209,
           274,
           109,
           266,
           277,
           117,
           86,
           93,
           315,
           258,
           278,
           258,
           277,
           258,
           0,
           258,
           288,
           258,
           319,
           258,
           0,
           258,
           0,
           258,
           0,
           258,
           0,
           258,
           287,
           258,
           315,
           258,
           289,
           258,
           278,
           99,
           269,
           266,
           262,
           8,
           259,
           241,
           4,
           217,
           230,
           268,
           266,
           55,
           168,
           106,
           75,
           193,
           266,
           223,
           27,
           49,
           26,
           282,
           25,
           264,
           299,
           19,
           26,
           0,
           258,
           277,
           117,
           86,
           93,
           176,
           183,
           270,
           11,
           262,
           42,
           61,
           265,
       ]

       self.assertListEqual(lowerCamelCase						,   self.big_tokenizer.encode(lowerCamelCase	)	)

     @require_torch
     @slow
     def 	UpperCamelCase(					self	):
       import torch

       from transformers import ReformerConfig, ReformerModel

       # Build sequence
       _snake_case     		=				list(self.big_tokenizer.get_vocab().keys()	)[:10]
       _snake_case     		=				" ".join(lowerCamelCase	)
       _snake_case     		=				self.big_tokenizer.encode_plus(lowerCamelCase						,   return_tensors="pt"	)
       _snake_case     		=				self.big_tokenizer.batch_encode_plus([sequence, sequence]						,   return_tensors="pt"	)

       _snake_case     		=				ReformerConfig()
       # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
       _snake_case     		=				encoded_sequence["input_ids"].shape
       _snake_case     		=				ReformerModel(lowerCamelCase	)

       # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
       assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

       with torch.no_grad():
         model(**lowerCamelCase	)
         model(**lowerCamelCase	)






     @slow
     def 	UpperCamelCase(					self	):
       # fmt: off
       _snake_case     		=				{"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
       # fmt: on

       # This tokenizer does not know some characters like ")".
       # That is the reason why we use very simple texts here.
       # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
       _snake_case     		=				[
           "This is a very simple sentence.",
           "The quick brown fox jumps over the lazy dog.",
       ]

       self.tokenizer_integration_test_util(
           expected_encoding=lowerCamelCase						,   model_name="google/reformer-crime-and-punishment"						,   revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"						,   padding=lowerCamelCase						,   sequences=lowerCamelCase						,   )


