
#
# This a `torch.distributed` diagnostics script that checks that all GPUs in the cluster (one or
# many nodes) can talk to each other via nccl and allocate gpu memory.
#
# To run first adjust the number of processes and nodes:
#
# python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# You may need to add --master_addr $MASTER_ADDR --master_port $MASTER_PORT if using a custom addr:port
#
# You can also use the rdzv API: --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT --rdzv_backend c10d
#
# use torch.distributed.launch instead of torch.distributed.run for torch < 1.9
#
# If you get a hanging in `barrier` calls you have some network issues, you may try to debug this with:
#
# NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# which should tell you what's going on behind the scenes.
#
#
# This script can be run via `srun` in the SLURM environment as well. Here is a SLURM script that
# runs on 2 nodes of 4 gpus per node:
#
# #SBATCH --job-name=test-nodes        # name
# #SBATCH --nodes=2                    # nodes
# #SBATCH --ntasks-per-node=1          # crucial - only 1 task per dist per node!
# #SBATCH --cpus-per-task=10           # number of cores per tasks
# #SBATCH --gres=gpu:4                 # number of gpus
# #SBATCH --time 0:05:00               # maximum execution time (HH:MM:SS)
# #SBATCH --output=%x-%j.out           # output file name
#
# GPUS_PER_NODE=4
# MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
# MASTER_PORT=6000
#
# srun --jobid $SLURM_JOBID bash -c 'python -m torch.distributed.run \
# --nproc_per_node $GPUS_PER_NODE --nnodes $SLURM_NNODES --node_rank $SLURM_PROCID \
# --master_addr $MASTER_ADDR --master_port $MASTER_PORT \
# torch-distributed-gpu-test.py'
#

import fcntl
import os
import socket

import torch
import torch.distributed as dist


def 							a__					(							*snake_case							):





  """simple docstring"""
  with open(snake_case       ,			'''r'''							) as fh:
    fcntl.flock(snake_case       ,			fcntl.LOCK_EX							)
    try:
      print(*snake_case							)
    finally:
      fcntl.flock(snake_case       ,			fcntl.LOCK_UN							)


lowercase_ 				=			int(os.environ["""LOCAL_RANK"""])
torch.cuda.set_device(local_rank)
lowercase_ 				=			torch.device("""cuda""", local_rank)
lowercase_ 				=			socket.gethostname()

lowercase_ 				=			f'''[{hostname}-{local_rank}]'''

try:
   # test distributed
   dist.init_process_group("""nccl""")
   dist.all_reduce(torch.ones(1).to(device), op=dist.ReduceOp.SUM)
   dist.barrier()

   # test cuda is available and can allocate memory
   torch.cuda.is_available()
   torch.ones(1).cuda(local_rank)

   # global rank
   lowercase_ 				=			dist.get_rank()
   lowercase_ 				=			dist.get_world_size()

   printflock(f'''{gpu} is OK (global rank: {rank}/{world_size})''')

   dist.barrier()
   if rank == 0:
      printflock(f'''pt={torch.__version__}, cuda={torch.version.cuda}, nccl={torch.cuda.nccl.version()}''')

except Exception:
   printflock(f'''{gpu} is broken''')
   raise




'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
      from PIL import Image

      from transformers import AlignProcessor, EfficientNetImageProcessor







@require_vision
class 							__magic_name__	(       unittest.TestCase ):

     """simple docstring"""






     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       int      =							tempfile.mkdtemp()

          __A	:       List[Any]      =							[
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          __A	:       Tuple      =							os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]		)
          with open(self.vocab_file			,     "w"			,     encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

          __A	:       List[str]      =							{
              "do_resize": True,
              "size": 20,
              "do_center_crop": True,
              "crop_size": 18,
              "do_normalize": True,
              "image_mean": [0.4814_5466, 0.457_8275, 0.4082_1073],
              "image_std": [0.2686_2954, 0.2613_0258, 0.2757_7711],
          }
          __A	:       Union[str, Any]      =							os.path.join(self.tmpdirname			,     lowerCamelCase		)
          with open(self.image_processor_file			,     "w"			,     encoding="utf-8"		) as fp:
               json.dump(lowerCamelCase			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     **lowerCamelCase		):


          '''simple docstring'''





          return BertTokenizer.from_pretrained(self.tmpdirname			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     **lowerCamelCase		):


          '''simple docstring'''





          return BertTokenizerFast.from_pretrained(self.tmpdirname			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     **lowerCamelCase		):


          '''simple docstring'''





          return EfficientNetImageProcessor.from_pretrained(self.tmpdirname			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          shutil.rmtree(self.tmpdirname		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       int      =							[np.random.randint(255			,     size=(3, 30, 400)			,     dtype=np.uinta		)]
          __A	:       int      =							[Image.fromarray(np.moveaxis(lowerCamelCase			,     0			,     -1		)		) for x in image_inputs]
          return image_inputs


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Union[str, Any]      =							self.get_tokenizer()
          __A	:       int      =							self.get_rust_tokenizer()
          __A	:       List[str]      =							self.get_image_processor()

          __A	:       List[Any]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)
          processor_slow.save_pretrained(self.tmpdirname		)
          __A	:       List[Any]      =							AlignProcessor.from_pretrained(self.tmpdirname			,     use_fast=lowerCamelCase		)

          __A	:       int      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)
          processor_fast.save_pretrained(self.tmpdirname		)
          __A	:       List[str]      =							AlignProcessor.from_pretrained(self.tmpdirname		)

          self.assertEqual(processor_slow.tokenizer.get_vocab()			,     tokenizer_slow.get_vocab()		)
          self.assertEqual(processor_fast.tokenizer.get_vocab()			,     tokenizer_fast.get_vocab()		)
          self.assertEqual(tokenizer_slow.get_vocab()			,     tokenizer_fast.get_vocab()		)
          self.assertIsInstance(processor_slow.tokenizer			,     lowerCamelCase		)
          self.assertIsInstance(processor_fast.tokenizer			,     lowerCamelCase		)

          self.assertEqual(processor_slow.image_processor.to_json_string()			,     image_processor.to_json_string()		)
          self.assertEqual(processor_fast.image_processor.to_json_string()			,     image_processor.to_json_string()		)
          self.assertIsInstance(processor_slow.image_processor			,     lowerCamelCase		)
          self.assertIsInstance(processor_fast.image_processor			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       List[Any]      =							AlignProcessor(tokenizer=self.get_tokenizer()			,     image_processor=self.get_image_processor()		)
          processor.save_pretrained(self.tmpdirname		)

          __A	:       List[str]      =							self.get_tokenizer(bos_token="(BOS)"			,     eos_token="(EOS)"		)
          __A	:       int      =							self.get_image_processor(do_normalize=lowerCamelCase			,     padding_value=1.0		)

          __A	:       Optional[int]      =							AlignProcessor.from_pretrained(
              self.tmpdirname			,     bos_token="(BOS)"			,     eos_token="(EOS)"			,     do_normalize=lowerCamelCase			,     padding_value=1.0		)

          self.assertEqual(processor.tokenizer.get_vocab()			,     tokenizer_add_kwargs.get_vocab()		)
          self.assertIsInstance(processor.tokenizer			,     lowerCamelCase		)

          self.assertEqual(processor.image_processor.to_json_string()			,     image_processor_add_kwargs.to_json_string()		)
          self.assertIsInstance(processor.image_processor			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							self.get_image_processor()
          __A	:       Optional[Any]      =							self.get_tokenizer()

          __A	:       List[str]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       int      =							self.prepare_image_inputs()

          __A	:       Optional[int]      =							image_processor(lowerCamelCase			,     return_tensors="np"		)
          __A	:       Dict      =							processor(images=lowerCamelCase			,     return_tensors="np"		)

          for key in input_image_proc.keys():
               self.assertAlmostEqual(input_image_proc[key].sum()			,     input_processor[key].sum()			,     delta=1E-2		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       str      =							self.get_image_processor()
          __A	:       int      =							self.get_tokenizer()

          __A	:       Optional[int]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       Optional[Any]      =							"lower newer"

          __A	:       Union[str, Any]      =							processor(text=lowerCamelCase		)

          __A	:       int      =							tokenizer(lowerCamelCase			,     padding="max_length"			,     max_length=64		)

          for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]			,     encoded_processor[key]		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[int]      =							self.get_image_processor()
          __A	:       Optional[Any]      =							self.get_tokenizer()

          __A	:       int      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       Tuple      =							"lower newer"
          __A	:       int      =							self.prepare_image_inputs()

          __A	:       str      =							processor(text=lowerCamelCase			,     images=lowerCamelCase		)

          self.assertListEqual(list(inputs.keys()		)			,     ["input_ids", "token_type_ids", "attention_mask", "pixel_values"]		)

          # test if it raises when no input is passed
          with pytest.raises(lowerCamelCase		):
               processor()


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Dict      =							self.get_image_processor()
          __A	:       Tuple      =							self.get_tokenizer()

          __A	:       Dict      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       Optional[int]      =							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          __A	:       Optional[int]      =							processor.batch_decode(lowerCamelCase		)
          __A	:       Optional[int]      =							tokenizer.batch_decode(lowerCamelCase		)

          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       str      =							self.get_image_processor()
          __A	:       Union[str, Any]      =							self.get_tokenizer()

          __A	:       Union[str, Any]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       int      =							"lower newer"
          __A	:       str      =							self.prepare_image_inputs()

          __A	:       Tuple      =							processor(text=lowerCamelCase			,     images=lowerCamelCase		)

          self.assertListEqual(list(inputs.keys()		)			,     processor.model_input_names		)

