





'''simple docstring'''




import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
  import os

  import jax.numpy as jnp
  from jax import jit

  from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
  from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

  lowercase__       : int         =       '''0.12'''  # assumed parallelism: 8


if is_torch_available():
  import torch






def 			_lowerCAmelCase     (    __snake_case    :					List[str]					,				__snake_case    :					Dict					,				__snake_case    :					str=None     )	->	Tuple:
     if rng is None:
          __A		:					str						=			random.Random()

     __A		:					Optional[Any]						=			1
     for dim in shape:
          total_dims *= dim

     __A		:					str						=			[]
     for _ in range(__snake_case     ):
          values.append(rng.randint(0					,				vocab_size - 1     )     )

     __A		:					List[Any]						=			np.array(__snake_case					,				dtype=jnp.intaa     ).reshape(__snake_case     )

     return output






def 			_lowerCAmelCase     (    __snake_case    :					Optional[Any]					,				__snake_case    :					Optional[int]=None     )	->	str:
     __A		:					Optional[int]						=			ids_tensor(__snake_case					,				vocab_size=2					,				rng=__snake_case     )
     # make sure that at least one token is attended to for each batch
     __A		:					Any						=			1
     return attn_mask


@require_flax
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			None
     lowerCAmelCase						     =			()


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A		:					List[str]						=			self.model_tester.prepare_config_and_inputs_for_common()

          # cut to half length & take max batch_size 3
          __A		:					str						=			2
          __A		:					Dict						=			inputs['input_ids'].shape[-1] // 2
          __A		:					Union[str, Any]						=			inputs['input_ids'][:max_batch_size, :sequence_length]

          __A		:					Dict						=			jnp.ones_like(_UpperCAmelCase)
          __A		:					List[str]						=			attention_mask[:max_batch_size, :sequence_length]

          # generate max 5 tokens
          __A		:					Optional[Any]						=			input_ids.shape[-1] + 5
          if config.eos_token_id is not None and config.pad_token_id is None:
               # hack to allow generate for models such as GPT2 as is done in `generate()`
               __A		:					Dict						=			config.eos_token_id
          return config, input_ids, attention_mask, max_length


     @is_pt_flax_cross_test
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					List[Any]						=			self._get_input_ids_and_config()
          __A		:					Optional[int]						=			False
          __A		:					List[Any]						=			max_length
          __A		:					Union[str, Any]						=			0

          for model_class in self.all_generative_model_classes:
               __A		:					List[Any]						=			model_class(_UpperCAmelCase)

               __A		:					Optional[int]						=			model_class.__name__[4:]  # Skip the "Flax" at the beginning
               __A		:					List[Any]						=			getattr(_UpperCAmelCase		,      _UpperCAmelCase)
               __A		:					str						=			pt_model_class(_UpperCAmelCase).eval()
               __A		:					int						=			load_flax_weights_in_pytorch_model(_UpperCAmelCase		,      flax_model.params)

               __A		:					Dict						=			flax_model.generate(_UpperCAmelCase).sequences
               __A		:					Optional[Any]						=			pt_model.generate(torch.tensor(_UpperCAmelCase		,      dtype=torch.long))

               if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
                    __A		:					Optional[int]						=			flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

               self.assertListEqual(pt_generation_outputs.numpy().tolist()		,      flax_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					List[Any]						=			self._get_input_ids_and_config()
          __A		:					Optional[Any]						=			False
          __A		:					str						=			max_length

          for model_class in self.all_generative_model_classes:
               __A		:					Union[str, Any]						=			model_class(_UpperCAmelCase)

               __A		:					int						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					Dict						=			jit(model.generate)
               __A		:					Any						=			jit_generate(_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					int						=			self._get_input_ids_and_config()
          __A		:					Any						=			True
          __A		:					int						=			max_length

          for model_class in self.all_generative_model_classes:
               __A		:					Union[str, Any]						=			model_class(_UpperCAmelCase)

               __A		:					Tuple						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					Dict						=			jit(model.generate)
               __A		:					List[Any]						=			jit_generate(_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					Union[str, Any]						=			self._get_input_ids_and_config()
          __A		:					int						=			False
          __A		:					int						=			max_length
          __A		:					str						=			2

          for model_class in self.all_generative_model_classes:
               __A		:					Optional[int]						=			model_class(_UpperCAmelCase)

               __A		:					Tuple						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					Dict						=			jit(model.generate)
               __A		:					str						=			jit_generate(_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					str						=			self._get_input_ids_and_config()
          __A		:					Optional[Any]						=			False
          __A		:					Optional[Any]						=			max_length
          __A		:					Any						=			2
          __A		:					Dict						=			2

          for model_class in self.all_generative_model_classes:
               __A		:					Dict						=			model_class(_UpperCAmelCase)

               __A		:					Any						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[0]		,      input_ids.shape[0] * config.num_return_sequences)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					Any						=			self._get_input_ids_and_config()
          __A		:					Tuple						=			True
          __A		:					Any						=			max_length
          __A		:					Optional[Any]						=			0.8
          __A		:					Any						=			10
          __A		:					Optional[int]						=			0.3
          __A		:					str						=			1
          __A		:					Dict						=			8
          __A		:					str						=			9

          for model_class in self.all_generative_model_classes:
               __A		:					Dict						=			model_class(_UpperCAmelCase)

               __A		:					Tuple						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					List[Any]						=			jit(model.generate)
               __A		:					List[Any]						=			jit_generate(_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					Optional[Any]						=			self._get_input_ids_and_config()
          __A		:					Union[str, Any]						=			max_length
          __A		:					Any						=			1
          __A		:					str						=			8
          __A		:					Dict						=			9

          for model_class in self.all_generative_model_classes:
               __A		:					int						=			model_class(_UpperCAmelCase)

               __A		:					Any						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					str						=			jit(model.generate)
               __A		:					int						=			jit_generate(_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					Any						=			self._get_input_ids_and_config()
          __A		:					str						=			max_length
          __A		:					Tuple						=			2
          __A		:					Optional[Any]						=			1
          __A		:					List[Any]						=			8
          __A		:					str						=			9

          for model_class in self.all_generative_model_classes:
               __A		:					Tuple						=			model_class(_UpperCAmelCase)

               __A		:					Optional[int]						=			model.generate(_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					Union[str, Any]						=			jit(model.generate)
               __A		:					str						=			jit_generate(_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					List[Any]						=			self._get_input_ids_and_config()

          # pad attention mask on the left
          __A		:					Any						=			attention_mask.at[(0, 0)].set(0)

          __A		:					Optional[Any]						=			False
          __A		:					Union[str, Any]						=			max_length

          for model_class in self.all_generative_model_classes:
               __A		:					Any						=			model_class(_UpperCAmelCase)

               __A		:					Any						=			model.generate(_UpperCAmelCase		,      attention_mask=_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					Optional[int]						=			jit(model.generate)
               __A		:					Union[str, Any]						=			jit_generate(_UpperCAmelCase		,      attention_mask=_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					Optional[Any]						=			self._get_input_ids_and_config()

          # pad attention mask on the left
          __A		:					Dict						=			attention_mask.at[(0, 0)].set(0)

          __A		:					str						=			True
          __A		:					Dict						=			max_length

          for model_class in self.all_generative_model_classes:
               __A		:					Any						=			model_class(_UpperCAmelCase)

               __A		:					List[str]						=			model.generate(_UpperCAmelCase		,      attention_mask=_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					Optional[Any]						=			jit(model.generate)
               __A		:					Union[str, Any]						=			jit_generate(_UpperCAmelCase		,      attention_mask=_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A					,__A					,__A		:					Optional[int]						=			self._get_input_ids_and_config()

          # pad attention mask on the left
          __A		:					List[str]						=			attention_mask.at[(0, 0)].set(0)

          __A		:					int						=			2
          __A		:					str						=			max_length

          for model_class in self.all_generative_model_classes:
               __A		:					str						=			model_class(_UpperCAmelCase)

               __A		:					Optional[Any]						=			model.generate(_UpperCAmelCase		,      attention_mask=_UpperCAmelCase).sequences
               self.assertEqual(generation_outputs.shape[-1]		,      _UpperCAmelCase)

               __A		:					str						=			jit(model.generate)
               __A		:					List[Any]						=			jit_generate(_UpperCAmelCase		,      attention_mask=_UpperCAmelCase).sequences

               self.assertListEqual(generation_outputs.tolist()		,      jit_generation_outputs.tolist())




@require_flax
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[Any]						=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-bert')
          __A		:					List[str]						=			FlaxAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-bert-flax-only')

          __A		:					List[Any]						=			'Hello world'
          __A		:					Union[str, Any]						=			tokenizer(_UpperCAmelCase		,      return_tensors='np').input_ids

          # typos are quickly detected (the correct argument is `do_sample`)
          with self.assertRaisesRegex(_UpperCAmelCase		,      'do_samples'):
               model.generate(_UpperCAmelCase		,      do_samples=_UpperCAmelCase)

          # arbitrary arguments that will not be used anywhere are also not accepted
          with self.assertRaisesRegex(_UpperCAmelCase		,      'foo'):
               __A		:					List[str]						=			{'foo': 'bar'}
               model.generate(_UpperCAmelCase		,      **_UpperCAmelCase)


"""simple docstring"""

import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			__a					(						__magic_name__       ,	unittest.TestCase       ):





       """simple docstring"""


       __UpperCamelCase					:      Optional[Any]    =						LxmertTokenizer
       __UpperCamelCase					:      Optional[Any]    =						LxmertTokenizerFast
       __UpperCamelCase					:      str    =						True
       __UpperCamelCase					:      Any    =						True






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        super().setUp()

        lowerCAmelCase__   :				Tuple 			=				[
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        lowerCAmelCase__   :				Dict 			=				os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES["vocab_file"]   )
        with open(self.vocab_file			,					"w"			,					encoding="utf-8"   ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[Any] 			=				"UNwant\u00E9d,running"
        lowerCAmelCase__   :				Union[str, Any] 			=				"unwanted, running"
        return input_text, output_text






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				str 			=				self.tokenizer_class(self.vocab_file   )

        lowerCAmelCase__   :				List[Any] 			=				tokenizer.tokenize("UNwant\u00E9d,running"   )
        self.assertListEqual(snake_case			,					["un", "##want", "##ed", ",", "runn", "##ing"]   )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case   )			,					[7, 4, 5, 10, 8, 9]   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        if not self.test_rust_tokenizer:
         return

        lowerCAmelCase__   :				int 			=				self.get_tokenizer()
        lowerCAmelCase__   :				List[str] 			=				self.get_rust_tokenizer()

        lowerCAmelCase__   :				Optional[Any] 			=				"I was born in 92000, and this is falsé."

        lowerCAmelCase__   :				int 			=				tokenizer.tokenize(snake_case   )
        lowerCAmelCase__   :				int 			=				rust_tokenizer.tokenize(snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )

        lowerCAmelCase__   :				str 			=				tokenizer.encode(snake_case			,					add_special_tokens=snake_case   )
        lowerCAmelCase__   :				List[str] 			=				rust_tokenizer.encode(snake_case			,					add_special_tokens=snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )

        lowerCAmelCase__   :				Tuple 			=				self.get_rust_tokenizer()
        lowerCAmelCase__   :				Tuple 			=				tokenizer.encode(snake_case   )
        lowerCAmelCase__   :				List[Any] 			=				rust_tokenizer.encode(snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )






