





import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import OPTConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
       import os

       # The slow tests are often failing with OOM error on GPU
       # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
       # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
       UpperCamelCase__          ='platform'

       import jax
       import jax.numpy as jnp

       from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer






def 			lowerCamelCase__	(__lowerCamelCase,				__lowerCamelCase,				__lowerCamelCase=None,				__lowerCamelCase=None		):
     if attention_mask is None:
          _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							np.where(input_ids != config.pad_token_id,				1,				0		)
     return {
         "input_ids": input_ids,
         "attention_mask": attention_mask,
     }





@require_flax
class 	lowerCAmelCase__:






       '''simple docstring'''


       def __init__(					self			,  __lowerCamelCase			,  __lowerCamelCase=1_3			,  __lowerCamelCase=7			,  __lowerCamelCase=True			,  __lowerCamelCase=False			,  __lowerCamelCase=9_9			,  __lowerCamelCase=1_6			,  __lowerCamelCase=2			,  __lowerCamelCase=4			,  __lowerCamelCase=4			,  __lowerCamelCase="gelu"			,  __lowerCamelCase=0.1			,  __lowerCamelCase=0.1			,  __lowerCamelCase=2_0			,  __lowerCamelCase=2			,  __lowerCamelCase=1			,  __lowerCamelCase=0			,  __lowerCamelCase=1_6			,  __lowerCamelCase=1_6			,  __lowerCamelCase=0.02			,  )	->   List[Any]:
            _SCREAMING_SNAKE_CASE				:		Any  			=							parent
            _SCREAMING_SNAKE_CASE				:		str  			=							batch_size
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							seq_length
            _SCREAMING_SNAKE_CASE				:		int  			=							is_training
            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							use_labels
            _SCREAMING_SNAKE_CASE				:		str  			=							vocab_size
            _SCREAMING_SNAKE_CASE				:		int  			=							hidden_size
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							num_hidden_layers
            _SCREAMING_SNAKE_CASE				:		str  			=							num_attention_heads
            _SCREAMING_SNAKE_CASE				:		Any  			=							intermediate_size
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							hidden_act
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							hidden_dropout_prob
            _SCREAMING_SNAKE_CASE				:		Any  			=							attention_probs_dropout_prob
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							max_position_embeddings
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							eos_token_id
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							pad_token_id
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							bos_token_id
            _SCREAMING_SNAKE_CASE				:		str  			=							embed_dim
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							word_embed_proj_dim
            _SCREAMING_SNAKE_CASE				:		str  			=							initializer_range
            _SCREAMING_SNAKE_CASE				:		str  			=							False




       def 			UpperCamelCase_	(					self							)	->   List[Any]:
            _SCREAMING_SNAKE_CASE				:		Dict  			=							np.clip(ids_tensor([self.batch_size, self.seq_length - 1]			,  self.vocab_size							)			,  3			,  self.vocab_size							)
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)			,  dtype=np.intaa							))			,  -1							)

            _SCREAMING_SNAKE_CASE				:		Any  			=							OPTConfig(
                vocab_size=self.vocab_size			,  hidden_size=self.hidden_size			,  num_hidden_layers=self.num_hidden_layers			,  num_attention_heads=self.num_attention_heads			,  ffn_dim=self.intermediate_size			,  dropout=self.hidden_dropout_prob			,  attention_dropout=self.attention_probs_dropout_prob			,  max_position_embeddings=self.max_position_embeddings			,  eos_token_id=self.eos_token_id			,  bos_token_id=self.bos_token_id			,  pad_token_id=self.pad_token_id			,  embed_dim=self.embed_dim			,  is_encoder_decoder=__lowerCamelCase			,  word_embed_proj_dim=self.word_embed_proj_dim			,  initializer_range=self.initializer_range			,  use_cache=__lowerCamelCase			,  )
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							prepare_opt_inputs_dict(__lowerCamelCase			,  __lowerCamelCase							)
            return config, inputs_dict




       def 			UpperCamelCase_	(					self							)	->   Any:
            _SCREAMING_SNAKE_CASE			,	_SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							self.prepare_config_and_inputs()
            return config, inputs_dict




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase			,  __lowerCamelCase							)	->   str:
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							2_0
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							model_class_name(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		Any  			=							inputs_dict["input_ids"]
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							inputs_dict["attention_mask"]

            _SCREAMING_SNAKE_CASE				:		Dict  			=							model.init_cache(input_ids.shape[0]			,  __lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							jnp.ones((input_ids.shape[0], max_length)			,  dtype="i4"							)

            _SCREAMING_SNAKE_CASE				:		Any  			=							jnp.broadcast_to(
                jnp.arange(input_ids.shape[-1] - 1							)[None, :]			,  (input_ids.shape[0], input_ids.shape[-1] - 1)			,  )
            _SCREAMING_SNAKE_CASE				:		int  			=							model(
                input_ids[:, :-1]			,  attention_mask=__lowerCamelCase			,  past_key_values=__lowerCamelCase			,  position_ids=__lowerCamelCase			,  )

            _SCREAMING_SNAKE_CASE				:		Dict  			=							jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]			,  dtype="i4"							)
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							model(
                input_ids[:, -1:]			,  attention_mask=__lowerCamelCase			,  past_key_values=outputs_cache.past_key_values			,  position_ids=__lowerCamelCase			,  )

            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							model(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		List[str]  			=							np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])							)							)
            self.parent.assertTrue(diff < 1E-3			,  msg=F"""Max diff is {diff}"""							)







       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase			,  __lowerCamelCase							)	->   Tuple:
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							2_0
            _SCREAMING_SNAKE_CASE				:		Any  			=							model_class_name(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE			,	_SCREAMING_SNAKE_CASE				:		List[Any]  			=							(
                inputs_dict["input_ids"],
                inputs_dict["attention_mask"],
            )

            _SCREAMING_SNAKE_CASE				:		Any  			=							jnp.concatenate(
                [
                    attention_mask,
                    jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])							),
                ]			,  axis=-1			,  )

            _SCREAMING_SNAKE_CASE				:		Any  			=							model.init_cache(input_ids.shape[0]			,  __lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							jnp.broadcast_to(
                jnp.arange(input_ids.shape[-1] - 1							)[None, :]			,  (input_ids.shape[0], input_ids.shape[-1] - 1)			,  )

            _SCREAMING_SNAKE_CASE				:		Any  			=							model(
                input_ids[:, :-1]			,  attention_mask=__lowerCamelCase			,  past_key_values=__lowerCamelCase			,  position_ids=__lowerCamelCase			,  )
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]			,  dtype="i4"							)
            _SCREAMING_SNAKE_CASE				:		Dict  			=							model(
                input_ids[:, -1:]			,  past_key_values=outputs_cache.past_key_values			,  attention_mask=__lowerCamelCase			,  position_ids=__lowerCamelCase			,  )

            _SCREAMING_SNAKE_CASE				:		Any  			=							model(__lowerCamelCase			,  attention_mask=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		Dict  			=							np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])							)							)
            self.parent.assertTrue(diff < 1E-3			,  msg=F"""Max diff is {diff}"""							)





@require_flax
class 	lowerCAmelCase__(       __lowercase     ,   unittest.TestCase     ,   __lowercase			):






       '''simple docstring'''


       __snake_case       	=			(FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
       __snake_case       	=			() if is_flax_available() else ()




       def 			UpperCamelCase_	(					self							)	->   str:
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							FlaxOPTModelTester(self							)




       def 			UpperCamelCase_	(					self							)	->   int:
            _SCREAMING_SNAKE_CASE			,	_SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							self.model_tester.prepare_config_and_inputs()
            for model_class in self.all_model_classes:
                 self.model_tester.check_use_cache_forward(__lowerCamelCase			,  __lowerCamelCase			,  __lowerCamelCase							)




       def 			UpperCamelCase_	(					self							)	->   Dict:
            _SCREAMING_SNAKE_CASE			,	_SCREAMING_SNAKE_CASE				:		str  			=							self.model_tester.prepare_config_and_inputs()
            for model_class in self.all_model_classes:
                 self.model_tester.check_use_cache_forward_with_attn_mask(__lowerCamelCase			,  __lowerCamelCase			,  __lowerCamelCase							)







       @slow
       def 			UpperCamelCase_	(					self							)	->   Dict:
            for model_class_name in self.all_model_classes:
                 _SCREAMING_SNAKE_CASE				:		List[Any]  			=							model_class_name.from_pretrained("facebook/opt-125m"							)
                 _SCREAMING_SNAKE_CASE				:		Tuple  			=							np.ones((1, 1)							) * model.config.eos_token_id
                 _SCREAMING_SNAKE_CASE				:		List[str]  			=							model(__lowerCamelCase							)
                 self.assertIsNotNone(__lowerCamelCase							)





@require_sentencepiece
@require_flax
class 	lowerCAmelCase__(       unittest.TestCase			):






       '''simple docstring'''


       @slow
       def 			UpperCamelCase_	(					self							)	->   Any:
            _SCREAMING_SNAKE_CASE				:		str  			=							FlaxOPTModel.from_pretrained("facebook/opt-350m"							)
            _SCREAMING_SNAKE_CASE				:		str  			=							jnp.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]							)
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							model(input_ids=__lowerCamelCase							).last_hidden_state
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							(1, 1_1, 5_1_2)
            self.assertEqual(output.shape			,  __lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							jnp.array(
                [[-0.2867, -1.9256, -0.3062], [-1.2711, -0.1337, -0.1897], [0.4109, 0.1187, -1.3142]]							)
            self.assertTrue(jnp.allclose(output[:, :3, :3]			,  __lowerCamelCase			,  atol=4E-2							)							)





@require_flax
@slow
class 	lowerCAmelCase__(       unittest.TestCase			):






       '''simple docstring'''


       def 			UpperCamelCase_	(					self							)	->   int:
            super().setUp()
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							"facebook/opt-350m"







       def 			UpperCamelCase_	(					self							)	->   List[str]:
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							FlaxOPTForCausalLM.from_pretrained(self.path_model							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							GPTaTokenizer.from_pretrained(self.path_model							)

            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							[
                "Today is a beautiful day and I want to",
                "In the city of",
                "Paris is the capital of France and",
                "Computers and mobile phones have taken",
            ]
            # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							tokenizer(__lowerCamelCase			,  return_tensors="jax"			,  padding=__lowerCamelCase			,  add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							model(inputs.input_ids			,  attention_mask=inputs.attention_mask							)[0].mean(axis=-1							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							jnp.array(
                [
                    [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
                    [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
                    [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
                    [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
                ]							)
            self.assertTrue(jnp.allclose(__lowerCamelCase			,  __lowerCamelCase			,  atol=4E-2							)							)

            _SCREAMING_SNAKE_CASE				:		Tuple  			=							jax.jit(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Dict  			=							model(inputs.input_ids			,  attention_mask=inputs.attention_mask							)[0].mean(axis=-1							)
            self.assertTrue(jnp.allclose(__lowerCamelCase			,  __lowerCamelCase			,  atol=4E-2							)							)





@require_flax
@slow
class 	lowerCAmelCase__(       unittest.TestCase			):






       '''simple docstring'''


       @property
       def 			UpperCamelCase_	(					self							)	->   Optional[int]:
            return [
                "Today is a beautiful day and I want",
                "In the city of",
                "Paris is the capital of France and",
                "Computers and mobile phones have taken",
            ]




       def 			UpperCamelCase_	(					self							)	->   str:
            _SCREAMING_SNAKE_CASE				:		int  			=							"facebook/opt-125m"

            _SCREAMING_SNAKE_CASE				:		Dict  			=							[
                "Today is a beautiful day and I want to",
                "In the city of New York, the city",
                "Paris is the capital of France and the capital",
                "Computers and mobile phones have taken over the",
            ]

            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							[]

            _SCREAMING_SNAKE_CASE				:		List[str]  			=							FlaxOPTForCausalLM.from_pretrained(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							GPTaTokenizer.from_pretrained(__lowerCamelCase							)

            for prompt in self.prompts:
                 _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							tokenizer(__lowerCamelCase			,  return_tensors="jax"							).input_ids

                 _SCREAMING_SNAKE_CASE				:		str  			=							model.generate(__lowerCamelCase			,  max_length=1_0							)
                 _SCREAMING_SNAKE_CASE				:		Dict  			=							generated_ids[0]

                 _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							tokenizer.batch_decode(__lowerCamelCase			,  skip_special_tokens=__lowerCamelCase							)
                 predicted_outputs += generated_string

            self.assertListEqual(__lowerCamelCase			,  __lowerCamelCase							)




       def 			UpperCamelCase_	(					self							)	->   Tuple:
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							"facebook/opt-350m"

            _SCREAMING_SNAKE_CASE				:		Dict  			=							[
                "Today is a beautiful day and I want to",
                "In the city of San Francisco, the city",
                "Paris is the capital of France and the capital",
                "Computers and mobile phones have taken over the",
            ]

            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							[]
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							FlaxOPTForCausalLM.from_pretrained(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		int  			=							GPTaTokenizer.from_pretrained(__lowerCamelCase							)

            for prompt in self.prompts:
                 _SCREAMING_SNAKE_CASE				:		List[str]  			=							tokenizer(__lowerCamelCase			,  return_tensors="jax"							).input_ids

                 _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							model.generate(__lowerCamelCase			,  max_length=1_0							)
                 _SCREAMING_SNAKE_CASE				:		Tuple  			=							generated_ids[0]

                 _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							tokenizer.batch_decode(__lowerCamelCase			,  skip_special_tokens=__lowerCamelCase							)
                 predicted_outputs += generated_string

            self.assertListEqual(__lowerCamelCase			,  __lowerCamelCase							)




       def 			UpperCamelCase_	(					self							)	->   List[str]:
            _SCREAMING_SNAKE_CASE				:		Dict  			=							"facebook/opt-125m"
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							[
                "Today is a beautiful day and I want to thank",
                "In the city of Rome Canaver Canaver Canaver Canaver",
            ]
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							FlaxOPTForCausalLM.from_pretrained(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							GPTaTokenizer.from_pretrained(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							tokenizer(
                [
                    "Today is a beautiful day and I want to",
                    "In the city of",
                ]			,  return_tensors="jax"			,  padding=__lowerCamelCase			,  )

            _SCREAMING_SNAKE_CASE				:		List[str]  			=							jax.jit(model.generate							)

            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							jit_generate(inputs["input_ids"]			,  attention_mask=inputs["attention_mask"]							).sequences

            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							tokenizer.batch_decode(__lowerCamelCase			,  skip_special_tokens=__lowerCamelCase							)

            self.assertIsNotNone(__lowerCamelCase			,  __lowerCamelCase							)







       def 			UpperCamelCase_	(					self							)	->   int:
            _SCREAMING_SNAKE_CASE				:		int  			=							"facebook/opt-350m"

            _SCREAMING_SNAKE_CASE				:		int  			=							GPTaTokenizer.from_pretrained(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							FlaxOPTForCausalLM.from_pretrained(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		Dict  			=							"left"

            # use different length sentences to test batching
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							[
                "Hello, my dog is a little",
                "Today, I",
            ]

            _SCREAMING_SNAKE_CASE				:		List[str]  			=							tokenizer(__lowerCamelCase			,  return_tensors="jax"			,  padding=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Any  			=							inputs["input_ids"]

            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							model.generate(input_ids=__lowerCamelCase			,  attention_mask=inputs["attention_mask"]			,  trace=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		Any  			=							tokenizer(sentences[0]			,  return_tensors="jax"							).input_ids
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							model.generate(input_ids=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		int  			=							inputs_non_padded.shape[-1] - inputs["attention_mask"][-1].sum()
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							tokenizer(sentences[1]			,  return_tensors="jax"							).input_ids
            _SCREAMING_SNAKE_CASE				:		Any  			=							model.generate(input_ids=__lowerCamelCase			,  max_length=model.config.max_length - num_paddings							)

            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							tokenizer.batch_decode(outputs[0]			,  skip_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							tokenizer.decode(output_non_padded[0][0]			,  skip_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		str  			=							tokenizer.decode(output_padded[0][0]			,  skip_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		int  			=							[
                "Hello, my dog is a little bit of a dork.\nI'm a little bit",
                "Today, I was in the middle of a conversation with a friend about the",
            ]
            self.assertListEqual(__lowerCamelCase			,  __lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase			,  [non_padded_sentence, padded_sentence]							)






import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


UpperCamelCase__          =logging.get_logger(__name__)

if is_tf_available():
       import tensorflow as tf





@dataclass
class 	lowerCAmelCase__(       __lowercase			):






       '''simple docstring'''


       __snake_case       	=			'tf'
       __snake_case       	=			field(
           default=__lowercase     ,   metadata={'help': 'Name of TPU'}     ,   )

       __snake_case       	=			field(
           default=__lowercase     ,   metadata={'help': 'Zone of TPU'}     ,   )

       __snake_case       	=			field(
           default=__lowercase     ,   metadata={'help': 'Name of Cloud TPU-enabled project'}     ,   )

       __snake_case       	=			field(
           default=1.0     ,   metadata={'help': 'Power for the Polynomial decay LR scheduler.'}     ,   )

       __snake_case       	=			field(default=__lowercase     ,   metadata={'help': 'Whether to activate the XLA compilation or not'}			)




       @cached_property
       def 			UpperCamelCase_	(					self							)	->   Tuple["tf.distribute.Strategy", int]:
            requires_backends(self			,  ["tf"]							)
            logger.info("Tensorflow: setting up strategy"							)

            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							tf.config.list_physical_devices("GPU"							)

            # Set to float16 at first
            if self.fpaa:
                 tf.keras.mixed_precision.set_global_policy("mixed_float16"							)

            if self.no_cuda:
                 _SCREAMING_SNAKE_CASE				:		List[Any]  			=							tf.distribute.OneDeviceStrategy(device="/cpu:0"							)
            else:
                 try:
                      if self.tpu_name:
                           _SCREAMING_SNAKE_CASE				:		Any  			=							tf.distribute.cluster_resolver.TPUClusterResolver(
                               self.tpu_name			,  zone=self.tpu_zone			,  project=self.gcp_project							)
                      else:
                           _SCREAMING_SNAKE_CASE				:		str  			=							tf.distribute.cluster_resolver.TPUClusterResolver()
                 except ValueError:
                      if self.tpu_name:
                           raise RuntimeError(F"""Couldn't connect to TPU {self.tpu_name}!"""							)
                      else:
                           _SCREAMING_SNAKE_CASE				:		Any  			=							None

                 if tpu:
                      # Set to bfloat16 in case of TPU
                      if self.fpaa:
                           tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"							)

                      tf.config.experimental_connect_to_cluster(__lowerCamelCase							)
                      tf.tpu.experimental.initialize_tpu_system(__lowerCamelCase							)

                      _SCREAMING_SNAKE_CASE				:		List[str]  			=							tf.distribute.TPUStrategy(__lowerCamelCase							)

                 elif len(__lowerCamelCase							) == 0:
                      _SCREAMING_SNAKE_CASE				:		Dict  			=							tf.distribute.OneDeviceStrategy(device="/cpu:0"							)
                 elif len(__lowerCamelCase							) == 1:
                      _SCREAMING_SNAKE_CASE				:		Dict  			=							tf.distribute.OneDeviceStrategy(device="/gpu:0"							)
                 elif len(__lowerCamelCase							) > 1:
                      # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
                      _SCREAMING_SNAKE_CASE				:		Any  			=							tf.distribute.MirroredStrategy()
                 else:
                      raise ValueError("Cannot find the proper strategy, please check your environment properties."							)

            return strategy




       @property
       def 			UpperCamelCase_	(					self							)	->   "tf.distribute.Strategy":
            requires_backends(self			,  ["tf"]							)
            return self._setup_strategy




       @property
       def 			UpperCamelCase_	(					self							)	->   int:
            requires_backends(self			,  ["tf"]							)
            return self._setup_strategy.num_replicas_in_sync




       @property
       def 			UpperCamelCase_	(					self							)	->   List[Any]:
            return False  # TF Logging is handled by Keras not the Trainer




       @property
       def 			UpperCamelCase_	(					self							)	->   int:
            if self.per_gpu_train_batch_size:
                 logger.warning(
                     "Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future "
                     "version. Using `--per_device_train_batch_size` is preferred."							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							self.per_gpu_train_batch_size or self.per_device_train_batch_size
            return per_device_batch_size * self.n_replicas




       @property
       def 			UpperCamelCase_	(					self							)	->   int:
            if self.per_gpu_eval_batch_size:
                 logger.warning(
                     "Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future "
                     "version. Using `--per_device_eval_batch_size` is preferred."							)
            _SCREAMING_SNAKE_CASE				:		int  			=							self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
            return per_device_batch_size * self.n_replicas







       @property
       def 			UpperCamelCase_	(					self							)	->   int:
            requires_backends(self			,  ["tf"]							)
            warnings.warn(
                "The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead."			,  __lowerCamelCase			,  )
            return self._setup_strategy.num_replicas_in_sync