
'''simple docstring'''
import argparse
import time

import torch
import transformers
from measures_util import end_measure, log_measures, start_measure
from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer

from accelerate.utils import compute_module_sizes


lowerCAmelCase :  Tuple      =      {
    'gpt-j-6b': {'is_causal': True, 'model': 'sgugger/sharded-gpt-j-6B', 'tokenizer': 'EleutherAI/gpt-j-6B'},
    'gpt-neox': {'is_causal': True, 'model': 'EleutherAI/gpt-neox-20b'},
    'opt': {'is_causal': True, 'model': 'facebook/opt-30b'},
    'T0pp': {'is_causal': False, 'model': 'bigscience/T0pp', 'model_revision': 'sharded'},
}

lowerCAmelCase :  Dict      =      [
    'Hello, my name is',
    'Are unicorns real? Unicorns are',
    'For the first time in several years,',
    'My name is Julien and I am',
    'The goal of life is',
    'Whenever I\'m sad, I like to',
]







def 						A_(			):
							UpperCamelCase					       =			argparse.ArgumentParser(description='Run and time generations on a big model using Accelerate.')
							parser.add_argument('model_name'						,			type=A						,			default=A						,			help='The name of the model to try.')
							parser.add_argument(
							    '--tokenizer_name'						,			type=A						,			default=A						,			help='The name of the tokenizer (if different from the model.')
							parser.add_argument('--is_causal'						,			type=A						,			default=A						,			help='Whether or not the model is causal.')
							parser.add_argument(
							    '--model_revision'						,			type=A						,			default=A						,			help='The revision to use for the model checkpoint.')
							parser.add_argument('--torch_dtype'						,			type=A						,			default=A						,			help='The dtype for the model.')
							parser.add_argument('--disk_offload'						,			action='store_true')

							UpperCamelCase					       =			parser.parse_args()

							# Sanitize args
							if args.model_name in DEFAULT_MODELS:
														UpperCamelCase					       =			DEFAULT_MODELS[args.model_name]
														UpperCamelCase					       =			defaults['model']
														if args.tokenizer_name is None:
																					UpperCamelCase					       =			defaults.get('tokenizer'						,			args.model_name)
														if args.is_causal is None:
																					UpperCamelCase					       =			defaults['is_causal']
														if args.model_revision is None:
																					UpperCamelCase					       =			defaults.get('model_revision'						,			'main')

							if args.is_causal is None:
														raise ValueError('Could not infer the default for `--is_causal`, pass either True or False for it.')
							if args.tokenizer_name is None:
														UpperCamelCase					       =			args.model_name
							if args.model_revision is None:
														UpperCamelCase					       =			'main'

							return args







def 						A_(			):
							transformers.utils.logging.set_verbosity_error()
							UpperCamelCase					       =			parse_args()

							if args.torch_dtype is None:
														UpperCamelCase					       =			AutoConfig.from_pretrained(args.model_name)
														UpperCamelCase					       =			getattr(A						,			'torch_dtype'						,			torch.floataa)
							else:
														UpperCamelCase					       =			getattr(A						,			args.torch_dtype)
							UpperCamelCase					       =			AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
							UpperCamelCase					       =			{
							    'torch_dtype': torch_dtype,
							    'revision': args.model_revision,
							}
							if args.disk_offload:
														UpperCamelCase					       =			'tmp_offload'
														UpperCamelCase					       =			True

							UpperCamelCase					       =			start_measure()
							UpperCamelCase					       =			model_cls.from_pretrained(args.model_name						,			device_map='auto'						,			**A)
							UpperCamelCase					       =			end_measure(A)
							log_measures(A						,			'Model loading')

							UpperCamelCase					       =			compute_module_sizes(A)
							UpperCamelCase					       =			{v: 0 for v in model.hf_device_map.values()}
							for module, device in model.hf_device_map.items():
														device_size[device] += module_sizes[module]
							UpperCamelCase					       =			'\n'.join([f'''- {device}: {size // 2**20}MiB''' for device, size in device_size.items()])
							print(f'''\nTheoretical use:\n{message}''')

							UpperCamelCase					       =			AutoTokenizer.from_pretrained(args.tokenizer_name)

							UpperCamelCase					       =			start_measure()
							UpperCamelCase					       =			[]
							UpperCamelCase					       =			[]
							UpperCamelCase					       =			[]
							for prompt in PROMPTS:
														UpperCamelCase					       =			tokenizer(A						,			return_tensors='pt').to(0)
														UpperCamelCase					       =			inputs['input_ids'][0].tolist()
														UpperCamelCase					       =			time.time()
														UpperCamelCase					       =			model.generate(inputs['input_ids'])
														UpperCamelCase					       =			time.time()
														UpperCamelCase					       =			outputs[0].tolist()
														UpperCamelCase					       =			len(A) if outputs[: len(A)] != tokens else len(A) - len(A)
														UpperCamelCase					       =			after_generate - before_generate

														UpperCamelCase					       =			tokenizer.decode(A						,			skip_special_tokens=A)
														texts_outs.append(A)
														generation_times.append(A)
														gen_tokens.append(A)
														print(f'''Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n''')

							UpperCamelCase					       =			end_measure(A)
							log_measures(A						,			'Model generation')

							UpperCamelCase					       =			[gen / tok for gen, tok in zip(A						,			A)]
							UpperCamelCase					       =			sum(A) / len(A)
							print(f'''Average time of generation per token: {avg_gen:.2f}s''')
							print(f'''First generation (avg time per token): {generation_times_per_token[0]:.2f}s''')
							UpperCamelCase					       =			sum(generation_times_per_token[1:]) / (len(A) - 1)
							print(f'''Average time of generation per token (excluding the first): {avg_gen:.2f}s''')


if __name__ == "__main__":
						main()






'''simple docstring'''
import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm







def 						A_(			A		:						int						,			A		:						bool = False):
							if device_specific:
														seed += AcceleratorState().process_index
							random.seed(A)
							np.random.seed(A)
							torch.manual_seed(A)
							if is_xpu_available():
														torch.xpu.manual_seed_all(A)
							elif is_npu_available():
														torch.npu.manual_seed_all(A)
							else:
														torch.cuda.manual_seed_all(A)
							# ^^ safe to call this function even if cuda is not available
							if is_tpu_available():
														xm.set_rng_state(A)







def 						A_(			A		:						Optional[RNGType] = None						,			A		:						Optional[torch.Generator] = None):
							# Get the proper rng state
							if rng_type == RNGType.TORCH:
														UpperCamelCase					       =			torch.get_rng_state()
							elif rng_type == RNGType.CUDA:
														UpperCamelCase					       =			torch.cuda.get_rng_state()
							elif rng_type == RNGType.XLA:
														assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
														UpperCamelCase					       =			torch.tensor(xm.get_rng_state())
							elif rng_type == RNGType.NPU:
														assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
														UpperCamelCase					       =			torch.npu.get_rng_state()
							elif rng_type == RNGType.XPU:
														assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
														UpperCamelCase					       =			torch.xpu.get_rng_state()
							elif rng_type == RNGType.GENERATOR:
														assert generator is not None, "Need a generator to synchronize its seed."
														UpperCamelCase					       =			generator.get_state()

							# Broadcast the rng state from device 0 to other devices
							UpperCamelCase					       =			AcceleratorState()
							if state.distributed_type == DistributedType.TPU:
														UpperCamelCase					       =			rng_state.to(xm.xla_device())
														xm.collective_broadcast([rng_state])
														xm.mark_step()
														UpperCamelCase					       =			rng_state.cpu()
							elif (
							    state.distributed_type in CUDA_DISTRIBUTED_TYPES
							    or state.distributed_type == DistributedType.MULTI_NPU
							    or state.distributed_type == DistributedType.MULTI_XPU
							):
														UpperCamelCase					       =			rng_state.to(state.device)
														torch.distributed.broadcast(A						,			0)
														UpperCamelCase					       =			rng_state.cpu()
							elif state.distributed_type == DistributedType.MULTI_CPU:
														torch.distributed.broadcast(A						,			0)

							# Set the broadcast rng state
							if rng_type == RNGType.TORCH:
														torch.set_rng_state(A)
							elif rng_type == RNGType.CUDA:
														torch.cuda.set_rng_state(A)
							elif rng_type == RNGType.NPU:
														torch.npu.set_rng_state(A)
							elif rng_type == RNGType.XPU:
														torch.xpu.set_rng_state(A)
							elif rng_type == RNGType.XLA:
														xm.set_rng_state(rng_state.item())
							elif rng_type == RNGType.GENERATOR:
														generator.set_state(A)







def 						A_(			A		:						List[Union[str, RNGType]]						,			A		:						Optional[torch.Generator] = None):
							for rng_type in rng_types:
														synchronize_rng_state(RNGType(A)						,			generator=A)




