

from copy import deepcopy

import torch
import torch.nn.functional as F
from torch.optim import AdamW
from torch.optim.lr_scheduler import LambdaLR
from torch.utils.data import DataLoader

from accelerate.accelerator import Accelerator
from accelerate.state import GradientState
from accelerate.test_utils import RegressionDataset, RegressionModel
from accelerate.utils import DistributedType, is_torch_version, set_seed



def 			_A	(      SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      ):
	for param, grad_param in zip(model_a.parameters()	,model_b.parameters()      ):
		if not param.requires_grad:
			continue
		if not did_step:
			# Grads should not be in sync
			assert (
			    torch.allclose(param.grad	,grad_param.grad      ) is False
			), f"Gradients in sync when they should not be at iteration {iteration}:\nmodel_a grad ({param.grad}) == model_b grad ({grad_param.grad})"
		else:
			# Grads should be in sync
			assert (
			    torch.allclose(param.grad	,grad_param.grad      ) is True
			), f"Gradients not in sync when they should be at iteration {iteration}:\nmodel_a grad ({param.grad}) != model_b grad ({grad_param.grad})"



def 			_A	(      SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE=True      ):
	model.train()
	UpperCAmelCase__:	Any									= model(SCREAMING_SNAKE_CASE      )
	UpperCAmelCase__:	Dict									= F.mse_loss(SCREAMING_SNAKE_CASE	,target.to(output.device      )      )
	if not do_backward:
		loss /= accelerator.gradient_accumulation_steps
		loss.backward()
	else:
		accelerator.backward(SCREAMING_SNAKE_CASE      )



def 			_A	(      SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE=False      ):
	set_seed(4_2      )
	UpperCAmelCase__:	List[str]									= RegressionModel()
	UpperCAmelCase__:	Optional[int]									= deepcopy(SCREAMING_SNAKE_CASE      )
	UpperCAmelCase__:	List[str]									= RegressionDataset(length=8_0      )
	UpperCAmelCase__:	int									= DataLoader(SCREAMING_SNAKE_CASE	,batch_size=1_6      )
	model.to(accelerator.device      )
	if sched:
		UpperCAmelCase__:	Dict									= AdamW(params=model.parameters()	,lr=1e-3      )
		UpperCAmelCase__:	List[str]									= AdamW(params=ddp_model.parameters()	,lr=1e-3      )
		UpperCAmelCase__:	int									= LambdaLR(SCREAMING_SNAKE_CASE	,lr_lambda=lambda SCREAMING_SNAKE_CASE      : epoch**0.65      )
		UpperCAmelCase__:	Optional[int]									= LambdaLR(SCREAMING_SNAKE_CASE	,lr_lambda=lambda SCREAMING_SNAKE_CASE      : epoch**0.65      )
	# Make a copy of `model`
	if sched:
		UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Any									= accelerator.prepare(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
	else:
		UpperCAmelCase__							,						UpperCAmelCase__:	Optional[int]									= accelerator.prepare(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
	if sched:
		return (model, opt, sched, dataloader, ddp_model, ddp_opt, ddp_sched)
	return model, ddp_model, dataloader



def 			_A	(      SCREAMING_SNAKE_CASE      ):
	# Test when on a single CPU or GPU that the context manager does nothing
	UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= get_training_setup(SCREAMING_SNAKE_CASE      )
	# Use a single batch
	UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= next(iter(SCREAMING_SNAKE_CASE      )      ).values()
	for iteration in range(3      ):
		# Gather the distributed inputs and targs for the base model
		UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= accelerator.gather((ddp_input, ddp_target)      )
		UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= input.to(accelerator.device      ), target.to(accelerator.device      )
		# Perform our initial ground truth step in non "DDP"
		step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		# Do "gradient accumulation" (noop)
		if iteration % 2 == 0:
			# Accumulate grads locally
			with accelerator.no_sync(SCREAMING_SNAKE_CASE      ):
				step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		else:
			# Sync grads
			step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )

		# Since `no_sync` is a noop, `ddp_model` and `model` grads should always be in sync
		check_model_parameters(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		for param, ddp_param in zip(model.parameters()	,ddp_model.parameters()      ):
			if not param.requires_grad:
				continue
			assert torch.allclose(
			    param.grad	,ddp_param.grad      ), f"Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})"

		# Shuffle ddp_input on each iteration
		torch.manual_seed(1_3_3_7 + iteration      )
		UpperCAmelCase__:	str									= ddp_input[torch.randperm(len(SCREAMING_SNAKE_CASE      )      )]



def 			_A	(      SCREAMING_SNAKE_CASE      ):
	# Test on distributed setup that context manager behaves properly
	UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= get_training_setup(SCREAMING_SNAKE_CASE      )
	# Use a single batch
	UpperCAmelCase__							,						UpperCAmelCase__:	Optional[Any]									= next(iter(SCREAMING_SNAKE_CASE      )      ).values()
	for iteration in range(3      ):
		# Gather the distributed inputs and targs for the base model
		UpperCAmelCase__							,						UpperCAmelCase__:	Optional[Any]									= accelerator.gather((ddp_input, ddp_target)      )
		UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= input.to(accelerator.device      ), target.to(accelerator.device      )
		# Perform our initial ground truth step in non "DDP"
		step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		# Do "gradient accumulation" (noop)
		if iteration % 2 == 0:
			# Accumulate grads locally
			with accelerator.no_sync(SCREAMING_SNAKE_CASE      ):
				step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		else:
			# Sync grads
			step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )

		# DDP model and model should only be in sync when not (iteration % 2 == 0)
		for param, ddp_param in zip(model.parameters()	,ddp_model.parameters()      ):
			if not param.requires_grad:
				continue
			if iteration % 2 == 0:
				# Grads should not be in sync
				assert (
				    torch.allclose(param.grad	,ddp_param.grad      ) is False
				), f"Gradients in sync when they should not be:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})"
			else:
				# Grads should be in sync
				assert (
				    torch.allclose(param.grad	,ddp_param.grad      ) is True
				), f"Gradients not in sync when they should be:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})"

        # Shuffle ddp_input on each iteration
		torch.manual_seed(1_3_3_7 + iteration      )
		UpperCAmelCase__:	Optional[int]									= ddp_input[torch.randperm(len(SCREAMING_SNAKE_CASE      )      )]



def 			_A	(      SCREAMING_SNAKE_CASE=False	,SCREAMING_SNAKE_CASE=False      ):
	UpperCAmelCase__:	List[Any]									= Accelerator(
	    split_batches=SCREAMING_SNAKE_CASE	,dispatch_batches=SCREAMING_SNAKE_CASE	,gradient_accumulation_steps=2      )
	# Test that context manager behaves properly
	UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Any									= get_training_setup(SCREAMING_SNAKE_CASE      )
	for iteration, batch in enumerate(SCREAMING_SNAKE_CASE      ):
		UpperCAmelCase__							,						UpperCAmelCase__:	Optional[Any]									= batch.values()
		# Gather the distributed inputs and targs for the base model
		UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= accelerator.gather((ddp_input, ddp_target)      )
		UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= input.to(accelerator.device      ), target.to(accelerator.device      )
		# Perform our initial ground truth step in non "DDP"
		step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		# Do "gradient accumulation" (noop)
		with accelerator.accumulate(SCREAMING_SNAKE_CASE      ):
			step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )

		# DDP model and model should only be in sync when not (iteration % 2 == 0)
		for param, ddp_param in zip(model.parameters()	,ddp_model.parameters()      ):
			if not param.requires_grad:
				continue
			if ((iteration + 1) % 2 == 0) or (iteration == len(SCREAMING_SNAKE_CASE      ) - 1):
				# Grads should be in sync
				assert (
				    torch.allclose(param.grad	,ddp_param.grad      ) is True
				), f"Gradients not in sync when they should be at iteration {iteration}:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})"
			else:
				# Grads should not be in sync
				assert (
				    torch.allclose(param.grad	,ddp_param.grad      ) is False
				), f"Gradients in sync when they should not be at iteration {iteration}:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})"

        # Shuffle ddp_input on each iteration
		torch.manual_seed(1_3_3_7 + iteration      )
		UpperCAmelCase__:	Optional[int]									= ddp_input[torch.randperm(len(SCREAMING_SNAKE_CASE      )      )]
	GradientState._reset_state()



def 			_A	(      SCREAMING_SNAKE_CASE=False	,SCREAMING_SNAKE_CASE=False      ):
	UpperCAmelCase__:	str									= Accelerator(
	    split_batches=SCREAMING_SNAKE_CASE	,dispatch_batches=SCREAMING_SNAKE_CASE	,gradient_accumulation_steps=2      )
	# Test that context manager behaves properly
	UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Any									= get_training_setup(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
	for iteration, batch in enumerate(SCREAMING_SNAKE_CASE      ):
		UpperCAmelCase__							,						UpperCAmelCase__:	Optional[int]									= batch.values()
		# Gather the distributed inputs and targs for the base model
		UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= accelerator.gather((ddp_input, ddp_target)      )
		UpperCAmelCase__							,						UpperCAmelCase__:	str									= input.to(accelerator.device      ), target.to(accelerator.device      )
		# Perform our initial ground truth step in non "DDP"
		model.train()
		ddp_model.train()
		step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		opt.step()

		if ((iteration + 1) % 2 == 0) or ((iteration + 1) == len(SCREAMING_SNAKE_CASE      )):
			if split_batches:
				sched.step()
			else:
				for _ in range(accelerator.num_processes      ):
					sched.step()
		opt.zero_grad()
		# Perform gradient accumulation under wrapper
		with accelerator.accumulate(SCREAMING_SNAKE_CASE      ):
			step_model(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
			ddp_opt.step()
			ddp_sched.step()
			ddp_opt.zero_grad()

		# Learning rates should be the same
		assert (
		    opt.param_groups[0]["lr"] == ddp_opt.param_groups[0]["lr"]
		), f"Learning rates found in each optimizer did not align\nopt: {opt.param_groups[0]['lr']}\nDDP opt: {ddp_opt.param_groups[0]['lr']}\n"
		UpperCAmelCase__:	Tuple									= (((iteration + 1) % 2) == 0) or ((iteration + 1) == len(SCREAMING_SNAKE_CASE      ))
		if accelerator.num_processes > 1:
			check_model_parameters(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
		# Shuffle ddp_input on each iteration
		torch.manual_seed(1_3_3_7 + iteration      )
	GradientState._reset_state()



def 			_A	(      ):
	UpperCAmelCase__:	Optional[int]									= Accelerator()

	UpperCAmelCase__:	Union[str, Any]									= RegressionDataset(length=8_0      )
	UpperCAmelCase__:	List[str]									= DataLoader(SCREAMING_SNAKE_CASE	,batch_size=1_6      )
	UpperCAmelCase__:	str									= RegressionDataset(length=9_6      )
	UpperCAmelCase__:	Optional[int]									= DataLoader(SCREAMING_SNAKE_CASE	,batch_size=1_6      )
	UpperCAmelCase__							,						UpperCAmelCase__:	List[Any]									= accelerator.prepare(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )
	assert accelerator.gradient_state.active_dataloader is None
	for iteration, _ in enumerate(SCREAMING_SNAKE_CASE      ):
		assert id(accelerator.gradient_state.active_dataloader      ) == id(SCREAMING_SNAKE_CASE      )
		if iteration < len(SCREAMING_SNAKE_CASE      ) - 1:
			assert not accelerator.gradient_state.end_of_dataloader
			if iteration == 1:
				for batch_num, _ in enumerate(SCREAMING_SNAKE_CASE      ):
					assert id(accelerator.gradient_state.active_dataloader      ) == id(SCREAMING_SNAKE_CASE      )
					if batch_num < len(SCREAMING_SNAKE_CASE      ) - 1:
						assert not accelerator.gradient_state.end_of_dataloader
					else:
						assert accelerator.gradient_state.end_of_dataloader
		else:
			assert accelerator.gradient_state.end_of_dataloader
	assert accelerator.gradient_state.active_dataloader is None



def 			_A	(      ):
	UpperCAmelCase__:	int									= Accelerator()
	UpperCAmelCase__:	int									= accelerator.state
	if state.local_process_index == 0:
		print("**Test `accumulate` gradient accumulation with dataloader break**"      )
	test_dataloader_break()
	if state.distributed_type == DistributedType.NO:
		if state.local_process_index == 0:
			print("**Test NOOP `no_sync` context manager**"      )
		test_noop_sync(SCREAMING_SNAKE_CASE      )
	if state.distributed_type in (DistributedType.MULTI_GPU, DistributedType.MULTI_CPU):
		if state.local_process_index == 0:
			print("**Test Distributed `no_sync` context manager**"      )
		test_distributed_sync(SCREAMING_SNAKE_CASE      )
	if state.distributed_type == DistributedType.MULTI_GPU:
		for split_batch in [True, False]:
			for dispatch_batches in [True, False]:
				if state.local_process_index == 0:
					print(
					    "**Test `accumulate` gradient accumulation, "	,f"`split_batches={split_batch}` and `dispatch_batches={dispatch_batches}`**"	,)
				test_gradient_accumulation(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )

    # Currently will break on torch 2.0 +, need to investigate why
	if is_torch_version("<"	,"2.0"      ) or state.distributed_type == DistributedType.NO:
		if state.local_process_index == 0:
			print(
			    "**Test `accumulate` gradient accumulation with optimizer and scheduler, "	,"`split_batches=False`, `dispatch_batches=False`**"	,)
		test_gradient_accumulation_with_opt_and_scheduler()
		if state.distributed_type == DistributedType.MULTI_GPU:
			for split_batch in [True, False]:
				for dispatch_batches in [True, False]:
					if not split_batch and not dispatch_batches:
						continue
					if state.local_process_index == 0:
						print(
						    "**Test `accumulate` gradient accumulation with optimizer and scheduler, "	,f"`split_batches={split_batch}` and `dispatch_batches={dispatch_batches}`**"	,)
					test_gradient_accumulation_with_opt_and_scheduler(SCREAMING_SNAKE_CASE	,SCREAMING_SNAKE_CASE      )



def 			_A	(      SCREAMING_SNAKE_CASE      ):
	# For xla_spawn (TPUs)
	main()


if __name__ == "__main__":
					main()


from typing import Callable, Dict, Optional, Tuple

import torch
from torch import nn
from torch.distributions import (
    AffineTransform,
    Distribution,
    Independent,
    NegativeBinomial,
    Normal,
    StudentT,
    TransformedDistribution,
)

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					def __init__(				self				,      lowerCamelCase__				,      lowerCamelCase__=None				,      lowerCamelCase__=None				,      lowerCamelCase__=0						):
						UpperCAmelCase__:	Any									= 1.0 if scale is None else scale
						UpperCAmelCase__:	int									= 0.0 if loc is None else loc

						super().__init__(lowerCamelCase__				,      [AffineTransform(loc=self.loc				,      scale=self.scale				,      event_dim=lowerCamelCase__						)]						)





					@property
					def    _UpperCAmelCase    (				self						):
						return self.base_dist.mean * self.scale + self.loc





					@property
					def    _UpperCAmelCase    (				self						):
						return self.base_dist.variance * self.scale**2





					@property
					def    _UpperCAmelCase    (				self						):
						return self.variance.sqrt()

class __UpperCamelCase     (				nn.Module		):






					'''simple docstring'''



					def __init__(				self				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      **lowerCamelCase__						):
						super().__init__(**lowerCamelCase__						)
						UpperCAmelCase__:	Any									= args_dim
						UpperCAmelCase__:	List[str]									= nn.ModuleList([nn.Linear(lowerCamelCase__				,      lowerCamelCase__						) for dim in args_dim.values()]						)
						UpperCAmelCase__:	Any									= domain_map





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	str									= [proj(lowerCamelCase__						) for proj in self.proj]

						return self.domain_map(*lowerCamelCase__						)

class __UpperCamelCase     (				nn.Module		):






					'''simple docstring'''



					def __init__(				self				,      lowerCamelCase__						):
						super().__init__()
						UpperCAmelCase__:	str									= function





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      *lowerCamelCase__						):
						return self.function(lowerCamelCase__				,      *lowerCamelCase__						)

class __UpperCamelCase     :






					'''simple docstring'''



					__magic_name__        =   42
					__magic_name__        =   42
					__magic_name__        =   42





					def __init__(				self				,      lowerCamelCase__ = 1						):
						UpperCAmelCase__:	str									= dim
						UpperCAmelCase__:	Any									= {k: dim * self.args_dim[k] for k in self.args_dim}





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						if self.dim == 1:
							return self.distribution_class(*lowerCamelCase__						)
						else:
							return Independent(self.distribution_class(*lowerCamelCase__						)				,      1						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__ = None				,      lowerCamelCase__ = None				,      ):
						UpperCAmelCase__:	Optional[Any]									= self._base_distribution(lowerCamelCase__						)
						if loc is None and scale is None:
							return distr
						else:
							return AffineTransformed(lowerCamelCase__				,      loc=lowerCamelCase__				,      scale=lowerCamelCase__				,      event_dim=self.event_dim						)





					@property
					def    _UpperCAmelCase    (				self						):
						return () if self.dim == 1 else (self.dim,)





					@property
					def    _UpperCAmelCase    (				self						):
						return len(self.event_shape						)





					@property
					def    _UpperCAmelCase    (				self						):
						return 0.0





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						return ParameterProjection(
						    in_features=lowerCamelCase__				,      args_dim=self.args_dim				,      domain_map=LambdaLayer(self.domain_map						)				,      )





					def    _UpperCAmelCase    (				self				,      *lowerCamelCase__						):
						raise NotImplementedError()





					@staticmethod
					def    _UpperCAmelCase    (				lowerCamelCase__						):
						return (x + torch.sqrt(torch.square(lowerCamelCase__						) + 4.0						)) / 2.0

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   {"df": 1, "loc": 1, "scale": 1}
					__magic_name__        =   StudentT





					@classmethod
					def    _UpperCAmelCase    (				cls				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__						):
						UpperCAmelCase__:	Dict									= cls.squareplus(lowerCamelCase__						).clamp_min(torch.finfo(scale.dtype						).eps						)
						UpperCAmelCase__:	List[Any]									= 2.0 + cls.squareplus(lowerCamelCase__						)
						return df.squeeze(-1						), loc.squeeze(-1						), scale.squeeze(-1						)

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   {"loc": 1, "scale": 1}
					__magic_name__        =   Normal





					@classmethod
					def    _UpperCAmelCase    (				cls				,      lowerCamelCase__				,      lowerCamelCase__						):
						UpperCAmelCase__:	str									= cls.squareplus(lowerCamelCase__						).clamp_min(torch.finfo(scale.dtype						).eps						)
						return loc.squeeze(-1						), scale.squeeze(-1						)

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   {"total_count": 1, "logits": 1}
					__magic_name__        =   NegativeBinomial





					@classmethod
					def    _UpperCAmelCase    (				cls				,      lowerCamelCase__				,      lowerCamelCase__						):
						UpperCAmelCase__:	Dict									= cls.squareplus(lowerCamelCase__						)
						return total_count.squeeze(-1						), logits.squeeze(-1						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__							,						UpperCAmelCase__:	str									= distr_args
						if self.dim == 1:
							return self.distribution_class(total_count=lowerCamelCase__				,      logits=lowerCamelCase__						)
						else:
							return Independent(self.distribution_class(total_count=lowerCamelCase__				,      logits=lowerCamelCase__						)				,      1						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__ = None				,      lowerCamelCase__ = None						):
						UpperCAmelCase__							,						UpperCAmelCase__:	List[str]									= distr_args

						if scale is not None:
							# See scaling property of Gamma.
							logits += scale.log()

						return self._base_distribution((total_count, logits)						)