






#
# This a `torch.distributed` diagnostics script that checks that all GPUs in the cluster (one or
# many nodes) can talk to each other via nccl and allocate gpu memory.
#
# To run first adjust the number of processes and nodes:
#
# python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# You may need to add --master_addr $MASTER_ADDR --master_port $MASTER_PORT if using a custom addr:port
#
# You can also use the rdzv API: --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT --rdzv_backend c10d
#
# use torch.distributed.launch instead of torch.distributed.run for torch < 1.9
#
# If you get a hanging in `barrier` calls you have some network issues, you may try to debug this with:
#
# NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# which should tell you what's going on behind the scenes.
#
#
# This script can be run via `srun` in the SLURM environment as well. Here is a SLURM script that
# runs on 2 nodes of 4 gpus per node:
#
# #SBATCH --job-name=test-nodes        # name
# #SBATCH --nodes=2                    # nodes
# #SBATCH --ntasks-per-node=1          # crucial - only 1 task per dist per node!
# #SBATCH --cpus-per-task=10           # number of cores per tasks
# #SBATCH --gres=gpu:4                 # number of gpus
# #SBATCH --time 0:05:00               # maximum execution time (HH:MM:SS)
# #SBATCH --output=%x-%j.out           # output file name
#
# GPUS_PER_NODE=4
# MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
# MASTER_PORT=6000
#
# srun --jobid $SLURM_JOBID bash -c 'python -m torch.distributed.run \
# --nproc_per_node $GPUS_PER_NODE --nnodes $SLURM_NNODES --node_rank $SLURM_PROCID \
# --master_addr $MASTER_ADDR --master_port $MASTER_PORT \
# torch-distributed-gpu-test.py'
#

import fcntl
import os
import socket

import torch
import torch.distributed as dist




def       _SCREAMING_SNAKE_CASE (							*__snake_case )				->						Optional[Any]:
       with open(UpperCamelCase_   ,      """r""" ) as fh:
              fcntl.flock(UpperCamelCase_   ,      fcntl.LOCK_EX )
              try:
                     print(*UpperCamelCase_ )
              finally:
                     fcntl.flock(UpperCamelCase_   ,      fcntl.LOCK_UN )


__a:     List[Any]   					=					int(os.environ['''LOCAL_RANK'''])
torch.cuda.set_device(local_rank)
__a:     List[str]   					=					torch.device('''cuda''', local_rank)
__a:     str   					=					socket.gethostname()

__a:     Optional[int]   					=					F"[{hostname}-{local_rank}]"

try:
  # test distributed
  dist.init_process_group('''nccl''')
  dist.all_reduce(torch.ones(1).to(device), op=dist.ReduceOp.SUM)
  dist.barrier()

  # test cuda is available and can allocate memory
  torch.cuda.is_available()
  torch.ones(1).cuda(local_rank)

  # global rank
  __a:     Optional[int]   					=					dist.get_rank()
  __a:     Optional[Any]   					=					dist.get_world_size()

  printflock(F"{gpu} is OK (global rank: {rank}/{world_size})")

  dist.barrier()
  if rank == 0:
    printflock(F"pt={torch.__version__}, cuda={torch.version.cuda}, nccl={torch.cuda.nccl.version()}")

except Exception:
  printflock(F"{gpu} is broken")
  raise
import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
     import os

     import jax.numpy as jnp
     from jax import jit

     from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
     from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

     _SCREAMING_SNAKE_CASE 		=      """0.12"""  # assumed parallelism: 8


if is_torch_available():
     import torch





def 			lowercase(	UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_=None       )		->	int:







 '''simple docstring'''
 if rng is None:
  UpperCamelCase			  =      random.Random()

 UpperCamelCase			  =      1
 for dim in shape:
  total_dims *= dim

 UpperCamelCase			  =      []
 for _ in range(UpperCamelCase_       ):
  values.append(rng.randint(0							,					vocab_size - 1       )       )

 UpperCamelCase			  =      np.array(UpperCamelCase_							,					dtype=jnp.intaa       ).reshape(UpperCamelCase_       )

 return output





def 			lowercase(	UpperCamelCase_							,					UpperCamelCase_=None       )		->	List[str]:







 '''simple docstring'''
 UpperCamelCase			  =      ids_tensor(UpperCamelCase_							,					vocab_size=2							,					rng=UpperCamelCase_       )
 # make sure that at least one token is attended to for each batch
 UpperCamelCase			  =      1
 return attn_mask
@require_flax
class SCREAMING_SNAKE_CASE_		:
   __lowerCAmelCase           =				None
   __lowerCAmelCase           =				()







   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase			  =      self.model_tester.prepare_config_and_inputs_for_common()

    # cut to half length & take max batch_size 3
    UpperCamelCase			  =      2
    UpperCamelCase			  =      inputs["""input_ids"""].shape[-1] // 2
    UpperCamelCase			  =      inputs["""input_ids"""][:max_batch_size, :sequence_length]

    UpperCamelCase			  =      jnp.ones_like(lowerCamelCase_				)
    UpperCamelCase			  =      attention_mask[:max_batch_size, :sequence_length]

    # generate max 5 tokens
    UpperCamelCase			  =      input_ids.shape[-1] + 5
    if config.eos_token_id is not None and config.pad_token_id is None:
     # hack to allow generate for models such as GPT2 as is done in `generate()`
     UpperCamelCase			  =      config.eos_token_id
    return config, input_ids, attention_mask, max_length







   @is_pt_flax_cross_test
   def 						lowerCamelCase_	(     self						:						Dict				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      False
    UpperCamelCase			  =      max_length
    UpperCamelCase			  =      0

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model_class.__name__[4:]  # Skip the "Flax" at the beginning
     UpperCamelCase			  =      getattr(lowerCamelCase_   ,   lowerCamelCase_				)
     UpperCamelCase			  =      pt_model_class(lowerCamelCase_				).eval()
     UpperCamelCase			  =      load_flax_weights_in_pytorch_model(lowerCamelCase_   ,   flax_model.params				)

     UpperCamelCase			  =      flax_model.generate(lowerCamelCase_				).sequences
     UpperCamelCase			  =      pt_model.generate(torch.tensor(lowerCamelCase_   ,   dtype=torch.long				)				)

     if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
      UpperCamelCase			  =      flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

     self.assertListEqual(pt_generation_outputs.numpy().tolist()   ,   flax_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						List[str]				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      False
    UpperCamelCase			  =      max_length

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						List[Any]				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      True
    UpperCamelCase			  =      max_length

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						Any				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      False
    UpperCamelCase			  =      max_length
    UpperCamelCase			  =      2

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      False
    UpperCamelCase			  =      max_length
    UpperCamelCase			  =      2
    UpperCamelCase			  =      2

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[0]   ,   input_ids.shape[0] * config.num_return_sequences				)







   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      True
    UpperCamelCase			  =      max_length
    UpperCamelCase			  =      0.8
    UpperCamelCase			  =      10
    UpperCamelCase			  =      0.3
    UpperCamelCase			  =      1
    UpperCamelCase			  =      8
    UpperCamelCase			  =      9

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						Optional[int]				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      max_length
    UpperCamelCase			  =      1
    UpperCamelCase			  =      8
    UpperCamelCase			  =      9

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						Any				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()
    UpperCamelCase			  =      max_length
    UpperCamelCase			  =      2
    UpperCamelCase			  =      1
    UpperCamelCase			  =      8
    UpperCamelCase			  =      9

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()

    # pad attention mask on the left
    UpperCamelCase			  =      attention_mask.at[(0, 0)].set(0				)

    UpperCamelCase			  =      False
    UpperCamelCase			  =      max_length

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_   ,   attention_mask=lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_   ,   attention_mask=lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()

    # pad attention mask on the left
    UpperCamelCase			  =      attention_mask.at[(0, 0)].set(0				)

    UpperCamelCase			  =      True
    UpperCamelCase			  =      max_length

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_   ,   attention_mask=lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_   ,   attention_mask=lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)







   def 						lowerCamelCase_	(     self						:						Any				):





    """simple docstring"""


    UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase  ,     UpperCamelCase			  =      self._get_input_ids_and_config()

    # pad attention mask on the left
    UpperCamelCase			  =      attention_mask.at[(0, 0)].set(0				)

    UpperCamelCase			  =      2
    UpperCamelCase			  =      max_length

    for model_class in self.all_generative_model_classes:
     UpperCamelCase			  =      model_class(lowerCamelCase_				)

     UpperCamelCase			  =      model.generate(lowerCamelCase_   ,   attention_mask=lowerCamelCase_				).sequences
     self.assertEqual(generation_outputs.shape[-1]   ,   lowerCamelCase_				)

     UpperCamelCase			  =      jit(model.generate				)
     UpperCamelCase			  =      jit_generate(lowerCamelCase_   ,   attention_mask=lowerCamelCase_				).sequences

     self.assertListEqual(generation_outputs.tolist()   ,   jit_generation_outputs.tolist()				)





@require_flax
class SCREAMING_SNAKE_CASE_		(      unittest.TestCase		):







   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    UpperCamelCase			  =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-bert"""				)
    UpperCamelCase			  =      FlaxAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""				)

    UpperCamelCase			  =      """Hello world"""
    UpperCamelCase			  =      tokenizer(lowerCamelCase_   ,   return_tensors="""np"""				).input_ids

    # typos are quickly detected (the correct argument is `do_sample`)
    with self.assertRaisesRegex(lowerCamelCase_   ,   """do_samples"""				):
     model.generate(lowerCamelCase_   ,   do_samples=lowerCamelCase_				)

    # arbitrary arguments that will not be used anywhere are also not accepted
    with self.assertRaisesRegex(lowerCamelCase_   ,   """foo"""				):
     UpperCamelCase			  =      {"""foo""": """bar"""}
     model.generate(lowerCamelCase_   ,   **lowerCamelCase_				)
