






#
# This a `torch.distributed` diagnostics script that checks that all GPUs in the cluster (one or
# many nodes) can talk to each other via nccl and allocate gpu memory.
#
# To run first adjust the number of processes and nodes:
#
# python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# You may need to add --master_addr $MASTER_ADDR --master_port $MASTER_PORT if using a custom addr:port
#
# You can also use the rdzv API: --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT --rdzv_backend c10d
#
# use torch.distributed.launch instead of torch.distributed.run for torch < 1.9
#
# If you get a hanging in `barrier` calls you have some network issues, you may try to debug this with:
#
# NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# which should tell you what's going on behind the scenes.
#
#
# This script can be run via `srun` in the SLURM environment as well. Here is a SLURM script that
# runs on 2 nodes of 4 gpus per node:
#
# #SBATCH --job-name=test-nodes        # name
# #SBATCH --nodes=2                    # nodes
# #SBATCH --ntasks-per-node=1          # crucial - only 1 task per dist per node!
# #SBATCH --cpus-per-task=10           # number of cores per tasks
# #SBATCH --gres=gpu:4                 # number of gpus
# #SBATCH --time 0:05:00               # maximum execution time (HH:MM:SS)
# #SBATCH --output=%x-%j.out           # output file name
#
# GPUS_PER_NODE=4
# MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
# MASTER_PORT=6000
#
# srun --jobid $SLURM_JOBID bash -c 'python -m torch.distributed.run \
# --nproc_per_node $GPUS_PER_NODE --nnodes $SLURM_NNODES --node_rank $SLURM_PROCID \
# --master_addr $MASTER_ADDR --master_port $MASTER_PORT \
# torch-distributed-gpu-test.py'
#

import fcntl
import os
import socket

import torch
import torch.distributed as dist

def      lowerCAmelCase__	(    *_a       :			int    ):







		with open(lowerCAmelCase__  ,      "r"    ) as fh:
				fcntl.flock(lowerCAmelCase__  ,      fcntl.LOCK_EX    )
				try:
						print(*lowerCAmelCase__    )
				finally:
						fcntl.flock(lowerCAmelCase__  ,      fcntl.LOCK_UN    )


lowercase  :							int			  =    int(os.environ['''LOCAL_RANK'''])
torch.cuda.set_device(local_rank)
lowercase  :							Dict			  =    torch.device('''cuda''', local_rank)
lowercase  :							Tuple			  =    socket.gethostname()

lowercase  :							Optional[Any]			  =    F"""[{hostname}-{local_rank}]"""

try:
							# test distributed
							dist.init_process_group('''nccl''')
							dist.all_reduce(torch.ones(1).to(device), op=dist.ReduceOp.SUM)
							dist.barrier()

							# test cuda is available and can allocate memory
							torch.cuda.is_available()
							torch.ones(1).cuda(local_rank)

							# global rank
							lowercase  :							Any			  =    dist.get_rank()
							lowercase  :							List[str]			  =    dist.get_world_size()

							printflock(F"""{gpu} is OK (global rank: {rank}/{world_size})""")

							dist.barrier()
							if rank == 0:
														printflock(F"""pt={torch.__version__}, cuda={torch.version.cuda}, nccl={torch.cuda.nccl.version()}""")

except Exception:
							printflock(F"""{gpu} is broken""")
							raise














import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase  :							List[Any]			  =    logging.get_logger(__name__)

lowercase  :							Any			  =    {
    '''asapp/sew-d-tiny-100k''': '''https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json''',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						A     :   Optional[int]						=					'sew-d'






						def __init__(	self		,				_SCREAMING_SNAKE_CASE=32		,				_SCREAMING_SNAKE_CASE=768		,				_SCREAMING_SNAKE_CASE=12		,				_SCREAMING_SNAKE_CASE=12		,				_SCREAMING_SNAKE_CASE=3072		,				_SCREAMING_SNAKE_CASE=2		,				_SCREAMING_SNAKE_CASE=512		,				_SCREAMING_SNAKE_CASE=256		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=("p2c", "c2p")		,				_SCREAMING_SNAKE_CASE="layer_norm"		,				_SCREAMING_SNAKE_CASE="gelu_python"		,				_SCREAMING_SNAKE_CASE=0.1		,				_SCREAMING_SNAKE_CASE=0.1		,				_SCREAMING_SNAKE_CASE=0.1		,				_SCREAMING_SNAKE_CASE=0.0		,				_SCREAMING_SNAKE_CASE=0.1		,				_SCREAMING_SNAKE_CASE=0.02		,				_SCREAMING_SNAKE_CASE=1e-7		,				_SCREAMING_SNAKE_CASE=1e-5		,				_SCREAMING_SNAKE_CASE="group"		,				_SCREAMING_SNAKE_CASE="gelu"		,				_SCREAMING_SNAKE_CASE=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)		,				_SCREAMING_SNAKE_CASE=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)		,				_SCREAMING_SNAKE_CASE=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE=128		,				_SCREAMING_SNAKE_CASE=16		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=0.05		,				_SCREAMING_SNAKE_CASE=10		,				_SCREAMING_SNAKE_CASE=2		,				_SCREAMING_SNAKE_CASE=0.0		,				_SCREAMING_SNAKE_CASE=10		,				_SCREAMING_SNAKE_CASE=0		,				_SCREAMING_SNAKE_CASE="mean"		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE=256		,				_SCREAMING_SNAKE_CASE=0		,				_SCREAMING_SNAKE_CASE=1		,				_SCREAMING_SNAKE_CASE=2		,				**_SCREAMING_SNAKE_CASE		,				) ->				List[Any]:
								super().__init__(**_SCREAMING_SNAKE_CASE		,				pad_token_id=_SCREAMING_SNAKE_CASE		,				bos_token_id=_SCREAMING_SNAKE_CASE		,				eos_token_id=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Optional[Any]						 =       hidden_size
								snake_case_				:							Any						 =       feat_extract_norm
								snake_case_				:							Union[str, Any]						 =       feat_extract_activation
								snake_case_				:							Any						 =       list(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       list(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Tuple						 =       list(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							List[Any]						 =       conv_bias
								snake_case_				:							Union[str, Any]						 =       num_conv_pos_embeddings
								snake_case_				:							Tuple						 =       num_conv_pos_embedding_groups
								snake_case_				:							Optional[int]						 =       len(self.conv_dim				)
								snake_case_				:							str						 =       num_hidden_layers
								snake_case_				:							List[Any]						 =       intermediate_size
								snake_case_				:							List[Any]						 =       squeeze_factor
								snake_case_				:							Tuple						 =       max_position_embeddings
								snake_case_				:							Optional[int]						 =       position_buckets
								snake_case_				:							Union[str, Any]						 =       share_att_key
								snake_case_				:							Optional[int]						 =       relative_attention
								snake_case_				:							List[str]						 =       norm_rel_ebd
								snake_case_				:							Tuple						 =       list(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       hidden_act
								snake_case_				:							Any						 =       num_attention_heads
								snake_case_				:							Dict						 =       hidden_dropout
								snake_case_				:							int						 =       attention_dropout
								snake_case_				:							Any						 =       activation_dropout
								snake_case_				:							Optional[Any]						 =       feat_proj_dropout
								snake_case_				:							Tuple						 =       final_dropout
								snake_case_				:							Tuple						 =       layer_norm_eps
								snake_case_				:							Tuple						 =       feature_layer_norm_eps
								snake_case_				:							Any						 =       initializer_range
								snake_case_				:							Optional[int]						 =       vocab_size

								if (
								    (len(self.conv_stride				) != self.num_feat_extract_layers)
								    or (len(self.conv_kernel				) != self.num_feat_extract_layers)
								    or (len(self.conv_dim				) != self.num_feat_extract_layers)
								):
										raise ValueError(
										    "Configuration for convolutional layers is incorrect."
										    "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"
										    f'''but is `len(config.conv_dim) = {len(self.conv_dim				)}`, `len(config.conv_stride)'''
										    f'''= {len(self.conv_stride				)}`, `len(config.conv_kernel) = {len(self.conv_kernel				)}`.'''				)

								# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
								snake_case_				:							List[str]						 =       apply_spec_augment
								snake_case_				:							List[Any]						 =       mask_time_prob
								snake_case_				:							str						 =       mask_time_length
								snake_case_				:							Optional[Any]						 =       mask_time_min_masks
								snake_case_				:							Optional[Any]						 =       mask_feature_prob
								snake_case_				:							str						 =       mask_feature_length
								snake_case_				:							Optional[int]						 =       mask_feature_min_masks

								# ctc loss
								snake_case_				:							Any						 =       ctc_loss_reduction
								snake_case_				:							Tuple						 =       ctc_zero_infinity

								# sequence classification
								snake_case_				:							int						 =       use_weighted_layer_sum
								snake_case_				:							List[Any]						 =       classifier_proj_size






						@property
						def       _lowerCAmelCase       (	self				) ->				Dict:
								return functools.reduce(operator.mul		,				self.conv_stride		,				1				)
