"""simple docstring"""





import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp



def   lowerCamelCase_()->     Optional[Any]:
							_SCREAMING_SNAKE_CASE		:    Dict      	=     ArgumentParser(
							    description=(
							        """PyTorch TPU distributed training launch """
							        """helper utility that will spawn up """
							        """multiple distributed processes"""
							    )	)

							# Optional arguments for the launch helper
							parser.add_argument("""--num_cores"""  ,						type=__SCREAMING_SNAKE_CASE  ,						default=1  ,						help="""Number of TPU cores to use (1 or 8)."""	)

							# positional
							parser.add_argument(
							    """training_script"""  ,						type=__SCREAMING_SNAKE_CASE  ,						help=(
							        """The full path to the single TPU training """
							        """program/script to be launched in parallel, """
							        """followed by all the arguments for the """
							        """training script"""
							    )  ,						)

							# rest from the training program
							parser.add_argument("""training_script_args"""  ,						nargs=__SCREAMING_SNAKE_CASE	)

							return parser.parse_args()



def   lowerCamelCase_()->     Union[str, Any]:
							_SCREAMING_SNAKE_CASE		:    Dict      	=     parse_args()

							# Import training_script as a module.
							_SCREAMING_SNAKE_CASE		:    Tuple      	=     Path(args.training_script	)
							sys.path.append(str(script_fpath.parent.resolve()	)	)
							_SCREAMING_SNAKE_CASE		:    Optional[int]      	=     script_fpath.stem
							_SCREAMING_SNAKE_CASE		:    List[Any]      	=     importlib.import_module(__SCREAMING_SNAKE_CASE	)

							# Patch sys.argv
							_SCREAMING_SNAKE_CASE		:    Tuple      	=     [args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores	)]
							xmp.spawn(mod._mp_fn  ,						args=()  ,						nprocs=args.num_cores	)


if __name__ == "__main__":
			main()




"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_            =    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            =    {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_				):
      __SCREAMING_SNAKE_CASE :				Optional[Any]				   =						"falcon"
      __SCREAMING_SNAKE_CASE :				List[Any]				   =						["past_key_values"]




      def __init__(				self ,							lowercase_=6_5_0_2_4 ,							lowercase_=4_5_4_4 ,							lowercase_=3_2 ,							lowercase_=7_1 ,							lowercase_=1E-5 ,							lowercase_=0.0_2 ,							lowercase_=True ,							lowercase_=0.0 ,							lowercase_=0.0 ,							lowercase_=None ,							lowercase_=False ,							lowercase_=False ,							lowercase_=True ,							lowercase_=True ,							lowercase_=False ,							lowercase_=1_1 ,							lowercase_=1_1 ,							**lowercase_ ,							)    ->       Any:
          UpperCAmelCase          =       vocab_size
          # Backward compatibility with n_embed kwarg
          UpperCAmelCase          =       kwargs.pop('n_embed' ,							lowercase_				)
          UpperCAmelCase          =       hidden_size if n_embed is None else n_embed
          UpperCAmelCase          =       num_hidden_layers
          UpperCAmelCase          =       num_attention_heads
          UpperCAmelCase          =       layer_norm_epsilon
          UpperCAmelCase          =       initializer_range
          UpperCAmelCase          =       use_cache
          UpperCAmelCase          =       hidden_dropout
          UpperCAmelCase          =       attention_dropout

          UpperCAmelCase          =       bos_token_id
          UpperCAmelCase          =       eos_token_id
          UpperCAmelCase          =       num_attention_heads if num_kv_heads is None else num_kv_heads
          UpperCAmelCase          =       alibi
          UpperCAmelCase          =       new_decoder_architecture
          UpperCAmelCase          =       multi_query  # Ignored when new_decoder_architecture is True
          UpperCAmelCase          =       parallel_attn
          UpperCAmelCase          =       bias

          super().__init__(bos_token_id=lowercase_ ,							eos_token_id=lowercase_ ,							**lowercase_				)




      @property
      def      a_   (				self				)    ->       List[Any]:
          return self.hidden_size // self.num_attention_heads




      @property
      def      a_   (				self				)    ->       List[str]:
          return not self.alibi



