

'''simple docstring'''
import warnings
from typing import List
from unittest.mock import Mock

import torch
from torch.utils.data import DataLoader, IterableDataset, TensorDataset

from accelerate.accelerator import Accelerator
from accelerate.utils.dataclasses import DistributedType


class   a__    (				__lowerCAmelCase    ):


   def __init__(					self     ,							_UpperCamelCase    ):





      """simple docstring"""





      _lowercase     : int     	=  data






   def __iter__(					self    ):





      """simple docstring"""





      for element in self.data:
         yield element







def    _A						(      snake_case=True	)     ->      Dict:
   _lowercase     : List[Any]     	=  Accelerator(even_batches=_UpperCamelCase	)
   assert accelerator.num_processes == 2, "this script expects that two GPUs are available"
   return accelerator







def    _A						(      snake_case						,				snake_case						,				snake_case						,				snake_case = False	)     ->      Dict:




   if iterable:
      _lowercase     : Dict     	=  DummyIterableDataset(torch.as_tensor(range(_UpperCamelCase	)	)	)
   else:
      _lowercase     : Optional[Any]     	=  TensorDataset(torch.as_tensor(range(_UpperCamelCase	)	)	)

   _lowercase     : Union[str, Any]     	=  DataLoader(_UpperCamelCase						,				batch_size=_UpperCamelCase	)
   _lowercase     : List[Any]     	=  accelerator.prepare(_UpperCamelCase	)

   return dl







def    _A						(      snake_case						,				snake_case						,				snake_case						,				snake_case						,				snake_case						,				)     ->      str:
   _lowercase     : Tuple     	=  create_dataloader(accelerator=_UpperCamelCase						,				dataset_size=_UpperCamelCase						,				batch_size=_UpperCamelCase	)

   _lowercase     : Tuple     	=  [len(batch[0]	) for batch in dl]

   if accelerator.process_index == 0:
      assert batch_sizes == process_0_expected_batch_sizes
   elif accelerator.process_index == 1:
      assert batch_sizes == process_1_expected_batch_sizes







def    _A						(      )     ->      Tuple:
   _lowercase     : str     	=  create_accelerator()

   # without padding, we would expect a different number of batches
   verify_dataloader_batch_sizes(
       _UpperCamelCase						,				dataset_size=3						,				batch_size=1						,				process_0_expected_batch_sizes=[1, 1]						,				process_1_expected_batch_sizes=[1, 1]						,				)

   # without padding, we would expect the same number of batches, but different sizes
   verify_dataloader_batch_sizes(
       _UpperCamelCase						,				dataset_size=7						,				batch_size=2						,				process_0_expected_batch_sizes=[2, 2]						,				process_1_expected_batch_sizes=[2, 2]						,				)







def    _A						(      )     ->      List[str]:
   _lowercase     : Dict     	=  create_accelerator(even_batches=_UpperCamelCase	)

   verify_dataloader_batch_sizes(
       _UpperCamelCase						,				dataset_size=3						,				batch_size=1						,				process_0_expected_batch_sizes=[1, 1]						,				process_1_expected_batch_sizes=[1]						,				)

   verify_dataloader_batch_sizes(
       _UpperCamelCase						,				dataset_size=7						,				batch_size=2						,				process_0_expected_batch_sizes=[2, 2]						,				process_1_expected_batch_sizes=[2, 1]						,				)







def    _A						(      )     ->      List[str]:
   _lowercase     : Any     	=  create_accelerator(even_batches=_UpperCamelCase	)

   _lowercase     : List[str]     	=  torch.nn.Linear(1						,				1	)
   _lowercase     : Dict     	=  accelerator.prepare(_UpperCamelCase	)

   _lowercase     : List[Any]     	=  create_dataloader(_UpperCamelCase						,				dataset_size=3						,				batch_size=1	)

   _lowercase     : str     	=  []
   with accelerator.join_uneven_inputs([ddp_model]	):
      for batch_idx, batch in enumerate(_UpperCamelCase	):
         _lowercase     : int     	=  ddp_model(batch[0].float()	)
         _lowercase     : Union[str, Any]     	=  output.sum()
         loss.backward()
         batch_idxs.append(_UpperCamelCase	)

   accelerator.wait_for_everyone()

   if accelerator.process_index == 0:
      assert batch_idxs == [0, 1]
   elif accelerator.process_index == 1:
      assert batch_idxs == [0]







def    _A						(      snake_case	)     ->      Any:




   with warnings.catch_warnings(record=_UpperCamelCase	) as w:
      with accelerator.join_uneven_inputs([Mock()]	):
         pass

      assert issubclass(w[-1].category						,				_UpperCamelCase	)
      assert "only supported for multi-GPU" in str(w[-1].message	)







def    _A						(      )     ->      Optional[int]:
   _lowercase     : Any     	=  True
   _lowercase     : Dict     	=  False
   _lowercase     : List[str]     	=  create_accelerator(even_batches=_UpperCamelCase	)
   _lowercase     : str     	=  torch.nn.Linear(1						,				1	)
   _lowercase     : List[Any]     	=  accelerator.prepare(_UpperCamelCase	)
   _lowercase     : int     	=  create_dataloader(_UpperCamelCase						,				dataset_size=3						,				batch_size=1	)
   _lowercase     : int     	=  create_dataloader(_UpperCamelCase						,				dataset_size=3						,				batch_size=1	)

   with accelerator.join_uneven_inputs([ddp_model]						,				even_batches=_UpperCamelCase	):
      _lowercase     : Dict     	=  train_dl.batch_sampler.even_batches
      _lowercase     : Optional[Any]     	=  valid_dl.batch_sampler.even_batches

   assert train_dl_overridden_value == overridden_even_batches
   assert valid_dl_overridden_value == overridden_even_batches
   assert train_dl.batch_sampler.even_batches == default_even_batches
   assert valid_dl.batch_sampler.even_batches == default_even_batches







def    _A						(      )     ->      List[str]:
   _lowercase     : int     	=  True
   _lowercase     : Tuple     	=  False
   _lowercase     : Optional[Any]     	=  create_accelerator(even_batches=_UpperCamelCase	)
   _lowercase     : Tuple     	=  torch.nn.Linear(1						,				1	)
   _lowercase     : Optional[Any]     	=  accelerator.prepare(_UpperCamelCase	)
   create_dataloader(_UpperCamelCase						,				dataset_size=3						,				batch_size=1						,				iterable=_UpperCamelCase	)
   _lowercase     : Optional[Any]     	=  create_dataloader(_UpperCamelCase						,				dataset_size=3						,				batch_size=1	)

   with warnings.catch_warnings():
      warnings.filterwarnings("ignore"	)
      try:
         with accelerator.join_uneven_inputs([ddp_model]						,				even_batches=_UpperCamelCase	):
            _lowercase     : int     	=  batch_dl.batch_sampler.even_batches
      except AttributeError:
         # ensure attribute error is not raised when processing iterable dl
         raise AssertionError

   assert batch_dl_overridden_value == overridden_even_batches
   assert batch_dl.batch_sampler.even_batches == default_even_batches







def    _A						(      )     ->      Any:
   _lowercase     : Any     	=  create_accelerator()
   _lowercase     : Tuple     	=  torch.nn.Linear(1						,				1	)
   _lowercase     : Dict     	=  accelerator.prepare(_UpperCamelCase	)
   create_dataloader(_UpperCamelCase						,				dataset_size=3						,				batch_size=1						,				iterable=_UpperCamelCase	)

   with warnings.catch_warnings(record=_UpperCamelCase	) as w:
      with accelerator.join_uneven_inputs([ddp_model]						,				even_batches=_UpperCamelCase	):
         pass

      assert issubclass(w[-1].category						,				_UpperCamelCase	)
      assert "only supported for map-style datasets" in str(w[-1].message	)







def    _A						(      )     ->      str:
   _lowercase     : List[Any]     	=  create_accelerator()

   accelerator.print("Test that even_batches variable ensures uniform batches across processes"	)
   test_default_ensures_even_batch_sizes()

   accelerator.print("Run tests with even_batches disabled"	)
   test_can_disable_even_batches()

   accelerator.print("Test joining uneven inputs"	)
   test_can_join_uneven_inputs()

   accelerator.print("Test overriding even_batches when joining uneven inputs"	)
   test_join_can_override_even_batches()

   accelerator.print("Test overriding even_batches for mixed dataloader types"	)
   test_join_can_override_for_mixed_type_dataloaders()

   accelerator.print("Test overriding even_batches raises a warning for iterable dataloaders"	)
   test_join_raises_warning_for_iterable_when_overriding_even_batches()

   accelerator.print("Test join with non DDP distributed raises warning"	)
   _lowercase     : Optional[int]     	=  accelerator.state.distributed_type
   _lowercase     : List[Any]     	=  DistributedType.FSDP
   test_join_raises_warning_for_non_ddp_distributed(_UpperCamelCase	)
   _lowercase     : int     	=  original_state


if __name__ == "__main__":
    main()












'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     int          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		"""falcon"""
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		["""past_key_values"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=65_024   ,_SCREAMING_SNAKE_CASE=4_544   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=71   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=11   ,_SCREAMING_SNAKE_CASE=11   ,**_SCREAMING_SNAKE_CASE   ,)				->     Dict:
          _snake_case				= vocab_size
          # Backward compatibility with n_embed kwarg
          _snake_case				= kwargs.pop("n_embed"   ,_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size if n_embed is None else n_embed
          _snake_case				= num_hidden_layers
          _snake_case				= num_attention_heads
          _snake_case				= layer_norm_epsilon
          _snake_case				= initializer_range
          _snake_case				= use_cache
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout

          _snake_case				= bos_token_id
          _snake_case				= eos_token_id
          _snake_case				= num_attention_heads if num_kv_heads is None else num_kv_heads
          _snake_case				= alibi
          _snake_case				= new_decoder_architecture
          _snake_case				= multi_query  # Ignored when new_decoder_architecture is True
          _snake_case				= parallel_attn
          _snake_case				= bias

          super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )






     @property
     def 							_lowercase      (					self       )				->     Union[str, Any]:
          return self.hidden_size // self.num_attention_heads






     @property
     def 							_lowercase      (					self       )				->     Optional[int]:
          return not self.alibi


