
from copy import deepcopy

import torch
import torch.nn.functional as F
from torch.optim import AdamW
from torch.optim.lr_scheduler import LambdaLR
from torch.utils.data import DataLoader

from accelerate.accelerator import Accelerator
from accelerate.state import GradientState
from accelerate.test_utils import RegressionDataset, RegressionModel
from accelerate.utils import DistributedType, is_torch_version, set_seed
def      UpperCAmelCase__				(							__magic_name__    :  Union[str, Any]				,	__magic_name__    :  Any				,	__magic_name__    :  str				,	__magic_name__    :  List[Any]	):


					'''simple docstring'''

					for param, grad_param in zip(model_a.parameters()				,	model_b.parameters()	):
										if not param.requires_grad:
															continue
										if not did_step:
															# Grads should not be in sync
															assert (
															    torch.allclose(param.grad				,	grad_param.grad	) is False
															), f'''Gradients in sync when they should not be at iteration {iteration}:\nmodel_a grad ({param.grad}) == model_b grad ({grad_param.grad})'''
										else:
															# Grads should be in sync
															assert (
															    torch.allclose(param.grad				,	grad_param.grad	) is True
															), f'''Gradients not in sync when they should be at iteration {iteration}:\nmodel_a grad ({param.grad}) != model_b grad ({grad_param.grad})'''
def      UpperCAmelCase__				(							__magic_name__    :  Optional[int]				,	__magic_name__    :  str				,	__magic_name__    :  List[str]				,	__magic_name__    :  Dict				,	__magic_name__    :  int=True	):


					'''simple docstring'''

					model.train()
					lowerCAmelCase		:       Any							=  model(__snake_case	)
					lowerCAmelCase		:       str							=  F.mse_loss(__snake_case				,	target.to(output.device	)	)
					if not do_backward:
										loss /= accelerator.gradient_accumulation_steps
										loss.backward()
					else:
										accelerator.backward(__snake_case	)
def      UpperCAmelCase__				(							__magic_name__    :  Any				,	__magic_name__    :  Dict=False	):


					'''simple docstring'''

					set_seed(42	)
					lowerCAmelCase		:       List[str]							=  RegressionModel()
					lowerCAmelCase		:       str							=  deepcopy(__snake_case	)
					lowerCAmelCase		:       Optional[int]							=  RegressionDataset(length=80	)
					lowerCAmelCase		:       Union[str, Any]							=  DataLoader(__snake_case				,	batch_size=16	)
					model.to(accelerator.device	)
					if sched:
										lowerCAmelCase		:       Any							=  AdamW(params=model.parameters()				,	lr=1e-3	)
										lowerCAmelCase		:       Tuple							=  AdamW(params=ddp_model.parameters()				,	lr=1e-3	)
										lowerCAmelCase		:       Tuple							=  LambdaLR(__snake_case				,	lr_lambda=lambda __magic_name__	: epoch**0.65	)
										lowerCAmelCase		:       Optional[Any]							=  LambdaLR(__snake_case				,	lr_lambda=lambda __magic_name__	: epoch**0.65	)
					# Make a copy of `model`
					if sched:
										lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase		:       Dict							=  accelerator.prepare(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
					else:
										lowerCAmelCase						,				lowerCAmelCase		:       Any							=  accelerator.prepare(__snake_case				,	__snake_case	)
					if sched:
										return (model, opt, sched, dataloader, ddp_model, ddp_opt, ddp_sched)
					return model, ddp_model, dataloader
def      UpperCAmelCase__				(							__magic_name__    :  Optional[int]	):


					'''simple docstring'''

					lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase		:       str							=  get_training_setup(__snake_case	)
					# Use a single batch
					lowerCAmelCase						,				lowerCAmelCase		:       Dict							=  next(iter(__snake_case	)	).values()
					for iteration in range(3	):
										# Gather the distributed inputs and targs for the base model
										lowerCAmelCase						,				lowerCAmelCase		:       List[Any]							=  accelerator.gather((ddp_input, ddp_target)	)
										lowerCAmelCase						,				lowerCAmelCase		:       List[Any]							=  input.to(accelerator.device	), target.to(accelerator.device	)
										# Perform our initial ground truth step in non "DDP"
										step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										# Do "gradient accumulation" (noop)
										if iteration % 2 == 0:
															# Accumulate grads locally
															with accelerator.no_sync(__snake_case	):
																				step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										else:
															# Sync grads
															step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)

										# Since `no_sync` is a noop, `ddp_model` and `model` grads should always be in sync
										check_model_parameters(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										for param, ddp_param in zip(model.parameters()				,	ddp_model.parameters()	):
															if not param.requires_grad:
																				continue
															assert torch.allclose(
															    param.grad				,	ddp_param.grad	), f'''Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})'''

										# Shuffle ddp_input on each iteration
										torch.manual_seed(13_37 + iteration	)
										lowerCAmelCase		:       Optional[Any]							=  ddp_input[torch.randperm(len(__snake_case	)	)]
def      UpperCAmelCase__				(							__magic_name__    :  str	):


					'''simple docstring'''

					lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase		:       Optional[Any]							=  get_training_setup(__snake_case	)
					# Use a single batch
					lowerCAmelCase						,				lowerCAmelCase		:       int							=  next(iter(__snake_case	)	).values()
					for iteration in range(3	):
										# Gather the distributed inputs and targs for the base model
										lowerCAmelCase						,				lowerCAmelCase		:       Tuple							=  accelerator.gather((ddp_input, ddp_target)	)
										lowerCAmelCase						,				lowerCAmelCase		:       Union[str, Any]							=  input.to(accelerator.device	), target.to(accelerator.device	)
										# Perform our initial ground truth step in non "DDP"
										step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										# Do "gradient accumulation" (noop)
										if iteration % 2 == 0:
															# Accumulate grads locally
															with accelerator.no_sync(__snake_case	):
																				step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										else:
															# Sync grads
															step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)

										# DDP model and model should only be in sync when not (iteration % 2 == 0)
										for param, ddp_param in zip(model.parameters()				,	ddp_model.parameters()	):
															if not param.requires_grad:
																				continue
															if iteration % 2 == 0:
																				# Grads should not be in sync
																				assert (
																				    torch.allclose(param.grad				,	ddp_param.grad	) is False
																				), f'''Gradients in sync when they should not be:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})'''
															else:
																				# Grads should be in sync
																				assert (
																				    torch.allclose(param.grad				,	ddp_param.grad	) is True
																				), f'''Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})'''

        # Shuffle ddp_input on each iteration
										torch.manual_seed(13_37 + iteration	)
										lowerCAmelCase		:       List[str]							=  ddp_input[torch.randperm(len(__snake_case	)	)]
def      UpperCAmelCase__				(							__magic_name__    :  Tuple=False				,	__magic_name__    :  Dict=False	):


					'''simple docstring'''

					lowerCAmelCase		:       List[str]							=  Accelerator(
					    split_batches=__snake_case				,	dispatch_batches=__snake_case				,	gradient_accumulation_steps=2	)
					# Test that context manager behaves properly
					lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase		:       List[Any]							=  get_training_setup(__snake_case	)
					for iteration, batch in enumerate(__snake_case	):
										lowerCAmelCase						,				lowerCAmelCase		:       Tuple							=  batch.values()
										# Gather the distributed inputs and targs for the base model
										lowerCAmelCase						,				lowerCAmelCase		:       Optional[Any]							=  accelerator.gather((ddp_input, ddp_target)	)
										lowerCAmelCase						,				lowerCAmelCase		:       Any							=  input.to(accelerator.device	), target.to(accelerator.device	)
										# Perform our initial ground truth step in non "DDP"
										step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										# Do "gradient accumulation" (noop)
										with accelerator.accumulate(__snake_case	):
															step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)

										# DDP model and model should only be in sync when not (iteration % 2 == 0)
										for param, ddp_param in zip(model.parameters()				,	ddp_model.parameters()	):
															if not param.requires_grad:
																				continue
															if ((iteration + 1) % 2 == 0) or (iteration == len(__snake_case	) - 1):
																				# Grads should be in sync
																				assert (
																				    torch.allclose(param.grad				,	ddp_param.grad	) is True
																				), f'''Gradients not in sync when they should be at iteration {iteration}:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})'''
															else:
																				# Grads should not be in sync
																				assert (
																				    torch.allclose(param.grad				,	ddp_param.grad	) is False
																				), f'''Gradients in sync when they should not be at iteration {iteration}:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})'''

        # Shuffle ddp_input on each iteration
										torch.manual_seed(13_37 + iteration	)
										lowerCAmelCase		:       int							=  ddp_input[torch.randperm(len(__snake_case	)	)]
					GradientState._reset_state()
def      UpperCAmelCase__				(							__magic_name__    :  Tuple=False				,	__magic_name__    :  Optional[int]=False	):


					'''simple docstring'''

					lowerCAmelCase		:       int							=  Accelerator(
					    split_batches=__snake_case				,	dispatch_batches=__snake_case				,	gradient_accumulation_steps=2	)
					# Test that context manager behaves properly
					lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase						,				lowerCAmelCase		:       List[str]							=  get_training_setup(__snake_case				,	__snake_case	)
					for iteration, batch in enumerate(__snake_case	):
										lowerCAmelCase						,				lowerCAmelCase		:       str							=  batch.values()
										# Gather the distributed inputs and targs for the base model
										lowerCAmelCase						,				lowerCAmelCase		:       str							=  accelerator.gather((ddp_input, ddp_target)	)
										lowerCAmelCase						,				lowerCAmelCase		:       int							=  input.to(accelerator.device	), target.to(accelerator.device	)
										# Perform our initial ground truth step in non "DDP"
										model.train()
										ddp_model.train()
										step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										opt.step()

										if ((iteration + 1) % 2 == 0) or ((iteration + 1) == len(__snake_case	)):
															if split_batches:
																				sched.step()
															else:
																				for _ in range(accelerator.num_processes	):
																									sched.step()
										opt.zero_grad()
										# Perform gradient accumulation under wrapper
										with accelerator.accumulate(__snake_case	):
															step_model(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
															ddp_opt.step()
															ddp_sched.step()
															ddp_opt.zero_grad()

										# Learning rates should be the same
										assert (
										    opt.param_groups[0]["lr"] == ddp_opt.param_groups[0]["lr"]
										), f'''Learning rates found in each optimizer did not align\nopt: {opt.param_groups[0]["lr"]}\nDDP opt: {ddp_opt.param_groups[0]["lr"]}\n'''
										lowerCAmelCase		:       Tuple							=  (((iteration + 1) % 2) == 0) or ((iteration + 1) == len(__snake_case	))
										if accelerator.num_processes > 1:
															check_model_parameters(__snake_case				,	__snake_case				,	__snake_case				,	__snake_case	)
										# Shuffle ddp_input on each iteration
										torch.manual_seed(13_37 + iteration	)
					GradientState._reset_state()
def      UpperCAmelCase__				(							):


					'''simple docstring'''

					lowerCAmelCase		:       Dict							=  Accelerator()

					lowerCAmelCase		:       List[str]							=  RegressionDataset(length=80	)
					lowerCAmelCase		:       str							=  DataLoader(__snake_case				,	batch_size=16	)
					lowerCAmelCase		:       int							=  RegressionDataset(length=96	)
					lowerCAmelCase		:       Tuple							=  DataLoader(__snake_case				,	batch_size=16	)
					lowerCAmelCase						,				lowerCAmelCase		:       Tuple							=  accelerator.prepare(__snake_case				,	__snake_case	)
					assert accelerator.gradient_state.active_dataloader is None
					for iteration, _ in enumerate(__snake_case	):
										assert id(accelerator.gradient_state.active_dataloader	) == id(__snake_case	)
										if iteration < len(__snake_case	) - 1:
															assert not accelerator.gradient_state.end_of_dataloader
															if iteration == 1:
																				for batch_num, _ in enumerate(__snake_case	):
																									assert id(accelerator.gradient_state.active_dataloader	) == id(__snake_case	)
																									if batch_num < len(__snake_case	) - 1:
																														assert not accelerator.gradient_state.end_of_dataloader
																									else:
																														assert accelerator.gradient_state.end_of_dataloader
										else:
															assert accelerator.gradient_state.end_of_dataloader
					assert accelerator.gradient_state.active_dataloader is None
def      UpperCAmelCase__				(							):


					'''simple docstring'''

					lowerCAmelCase		:       str							=  Accelerator()
					lowerCAmelCase		:       List[str]							=  accelerator.state
					if state.local_process_index == 0:
										print('''**Test `accumulate` gradient accumulation with dataloader break**'''	)
					test_dataloader_break()
					if state.distributed_type == DistributedType.NO:
										if state.local_process_index == 0:
															print('''**Test NOOP `no_sync` context manager**'''	)
										test_noop_sync(__snake_case	)
					if state.distributed_type in (DistributedType.MULTI_GPU, DistributedType.MULTI_CPU):
										if state.local_process_index == 0:
															print('''**Test Distributed `no_sync` context manager**'''	)
										test_distributed_sync(__snake_case	)
					if state.distributed_type == DistributedType.MULTI_GPU:
										for split_batch in [True, False]:
															for dispatch_batches in [True, False]:
																				if state.local_process_index == 0:
																									print(
																									    '''**Test `accumulate` gradient accumulation, '''				,	f'''`split_batches={split_batch}` and `dispatch_batches={dispatch_batches}`**'''				,	)
																				test_gradient_accumulation(__snake_case				,	__snake_case	)

    # Currently will break on torch 2.0 +, need to investigate why
					if is_torch_version('''<'''				,	'''2.0'''	) or state.distributed_type == DistributedType.NO:
										if state.local_process_index == 0:
															print(
															    '''**Test `accumulate` gradient accumulation with optimizer and scheduler, '''				,	'''`split_batches=False`, `dispatch_batches=False`**'''				,	)
										test_gradient_accumulation_with_opt_and_scheduler()
										if state.distributed_type == DistributedType.MULTI_GPU:
															for split_batch in [True, False]:
																				for dispatch_batches in [True, False]:
																									if not split_batch and not dispatch_batches:
																														continue
																									if state.local_process_index == 0:
																														print(
																														    '''**Test `accumulate` gradient accumulation with optimizer and scheduler, '''				,	f'''`split_batches={split_batch}` and `dispatch_batches={dispatch_batches}`**'''				,	)
																									test_gradient_accumulation_with_opt_and_scheduler(__snake_case				,	__snake_case	)
def      UpperCAmelCase__				(							__magic_name__    :  List[Any]	):


					'''simple docstring'''

					main()


if __name__ == "__main__":
		main()








from __future__ import annotations

import os
from typing import Any

import requests

lowerCamelCase      :       Tuple    =						'''https://api.github.com'''

# https://docs.github.com/en/free-pro-team@latest/rest/reference/users#get-the-authenticated-user
lowerCamelCase      :       int    =						BASE_URL + '''/user'''

# https://github.com/settings/tokens
lowerCamelCase      :       Any    =						os.environ.get('''USER_TOKEN''', '''''')





def    __lowerCAmelCase   (    __snake_case							):
     __lowerCAmelCase             =       {
         "Authorization": F"""token {auth_token}""",
         "Accept": "application/vnd.github.v3+json",
     }
     return requests.get(__snake_case       ,			headers=__snake_case							).json()


if __name__ == "__main__":  # pragma: no cover
 if USER_TOKEN:
  for key, value in fetch_github_info(USER_TOKEN).items():
   print(F'''{key}: {value}''')
 else:
  raise ValueError('''\'USER_TOKEN\' field cannot be empty.''')






