





'''simple docstring'''




import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)




def 						__lowerCamelCase			(	UpperCAmelCase_							)   ->Dict:
	if model.config.model_type == "gpt2":
		return model.transformer.h[0].mlp.c_fc
	return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
			import torch
			import torch.nn as nn






			class 				__snake_case     (					nn.Module							):





				def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_		)      ->		str:
					super().__init__()
					snake_case__	       =   module
					snake_case__	       =   nn.Sequential(
					    nn.Linear(module.in_features      ,       UpperCamelCase_      ,       bias=UpperCamelCase_		)      ,       nn.Linear(UpperCamelCase_      ,       module.out_features      ,       bias=UpperCamelCase_		)      ,       )
					snake_case__	       =   (2.0 / (5 * min(module.in_features      ,       module.out_features		))) ** 0.5
					nn.init.normal_(self.adapter[0].weight      ,       std=UpperCamelCase_		)
					nn.init.zeros_(self.adapter[1].weight		)
					self.adapter.to(module.weight.device		)





				def   _snake_case   (		self      ,       UpperCamelCase_      ,       *UpperCamelCase_      ,       **UpperCamelCase_		)      ->		List[Any]:
					return self.module(UpperCamelCase_      ,       *UpperCamelCase_      ,       **UpperCamelCase_		) + self.adapter(UpperCamelCase_		)






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				__snake_case     (					unittest.TestCase							):
	# We keep the constants inside the init function and model loading inside setUp function

	# We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
	# Therefore here we use only bloom-1b3 to test our module
	__lowerCAmelCase       						=  '''bigscience/bloom-1b7'''

	# Constant values
	__lowerCAmelCase       						=  2.109659552692574

	__lowerCAmelCase       						=  '''Hello my name is'''
	__lowerCAmelCase       						=  set()
	EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I'''							)
	EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n'''							)
	EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University'''							)
	__lowerCAmelCase       						=  10





	def   _snake_case   (		self		)      ->		List[Any]:
		# Models and tokenizer
		snake_case__	       =   AutoTokenizer.from_pretrained(self.model_name		)






class 				__snake_case     (					__magic_name__							):





	def   _snake_case   (		self		)      ->		Any:
		super().setUp()

		# Models and tokenizer
		snake_case__	       =   AutoModelForCausalLM.from_pretrained(
		    self.model_name      ,       torch_dtype=torch.floataa      ,       device_map='auto'		)
		snake_case__	       =   AutoModelForCausalLM.from_pretrained(self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)





	def   _snake_case   (		self		)      ->		str:
		del self.model_fpaa
		del self.model_abit

		gc.collect()
		torch.cuda.empty_cache()





	def   _snake_case   (		self		)      ->		Union[str, Any]:
		snake_case__	       =   self.model_abit.config

		self.assertTrue(hasattr(UpperCamelCase_      ,       'quantization_config'		)		)

		snake_case__	       =   config.to_dict()
		snake_case__	       =   config.to_diff_dict()

		snake_case__	       =   config.to_json_string()





	def   _snake_case   (		self		)      ->		Optional[Any]:
		from bitsandbytes.nn import Paramsabit

		snake_case__	       =   self.model_fpaa.get_memory_footprint()
		snake_case__	       =   self.model_abit.get_memory_footprint()

		self.assertAlmostEqual(mem_fpaa / mem_abit      ,       self.EXPECTED_RELATIVE_DIFFERENCE		)
		snake_case__	       =   get_some_linear_layer(self.model_abit		)
		self.assertTrue(linear.weight.__class__ == Paramsabit		)





	def   _snake_case   (		self		)      ->		int:
		from transformers import TaPreTrainedModel

		self.model_fpaa.get_memory_footprint()
		self.model_abit.get_memory_footprint()

		for name, module in self.model_abit.named_modules():
			if isinstance(UpperCamelCase_      ,       torch.nn.Linear		):
				if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
					# 4-bit parameters are packed in uint8 variables
					self.assertTrue(module.weight.dtype == torch.uinta		)





	def   _snake_case   (		self		)      ->		Optional[Any]:
		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		)
		snake_case__	       =   self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0		)      ,       max_new_tokens=10		)

		self.assertIn(self.tokenizer.decode(output_sequences[0]      ,       skip_special_tokens=UpperCamelCase_		)      ,       self.EXPECTED_OUTPUTS		)





	def   _snake_case   (		self		)      ->		Any:
		snake_case__	       =   BitsAndBytesConfig()
		snake_case__	       =   True

		snake_case__	       =   AutoModelForCausalLM.from_pretrained(
		    self.model_name      ,       quantization_config=UpperCamelCase_      ,       device_map='auto'		)

		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		)
		snake_case__	       =   model_abit_from_config.generate(
		    input_ids=encoded_input['input_ids'].to(0		)      ,       max_new_tokens=10		)

		self.assertIn(self.tokenizer.decode(output_sequences[0]      ,       skip_special_tokens=UpperCamelCase_		)      ,       self.EXPECTED_OUTPUTS		)





	def   _snake_case   (		self		)      ->		Dict:
		with self.assertRaises(UpperCamelCase_		), tempfile.TemporaryDirectory() as tmpdirname:
			self.model_abit.save_pretrained(UpperCamelCase_		)





	def   _snake_case   (		self		)      ->		Tuple:
		snake_case__	       =   BitsAndBytesConfig()

		with self.assertRaises(UpperCamelCase_		):
			snake_case__	       =   AutoModelForCausalLM.from_pretrained(
			    self.model_name      ,       quantization_config=UpperCamelCase_      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'      ,       bnb_abit_quant_type='nf4'      ,       )





	def   _snake_case   (		self		)      ->		Any:
		with self.assertRaises(UpperCamelCase_		):
			# Tries with `str`
			self.model_abit.to('cpu'		)

		with self.assertRaises(UpperCamelCase_		):
			# Tries with a `dtype``
			self.model_abit.to(torch.floataa		)

		with self.assertRaises(UpperCamelCase_		):
			# Tries with a `device`
			self.model_abit.to(torch.device('cuda:0'		)		)

		with self.assertRaises(UpperCamelCase_		):
			# Tries with a `device`
			self.model_abit.float()

		with self.assertRaises(UpperCamelCase_		):
			# Tries with a `device`
			self.model_abit.half()

		# Test if we did not break anything
		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		)

		snake_case__	       =   self.model_fpaa.to(torch.floataa		)
		snake_case__	       =   self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0		)      ,       max_new_tokens=10		)

		# Check this does not throw an error
		snake_case__	       =   self.model_fpaa.to('cpu'		)

		# Check this does not throw an error
		snake_case__	       =   self.model_fpaa.half()

		# Check this does not throw an error
		snake_case__	       =   self.model_fpaa.float()





	def   _snake_case   (		self		)      ->		Dict:
		snake_case__	       =   AutoModelForSeqaSeqLM.from_pretrained('t5-small'      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa		)






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				__snake_case     (					unittest.TestCase							):





	@classmethod
	def   _snake_case   (		cls		)      ->		Any:
		snake_case__	       =   't5-small'
		snake_case__	       =   'google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
		snake_case__	       =   AutoTokenizer.from_pretrained(cls.model_name		)
		snake_case__	       =   'Translate in German: Hello, my dog is cute'





	def   _snake_case   (		self		)      ->		List[str]:
		gc.collect()
		torch.cuda.empty_cache()





	def   _snake_case   (		self		)      ->		Union[str, Any]:
		from transformers import TaForConditionalGeneration

		snake_case__	       =   TaForConditionalGeneration._keep_in_fpaa_modules
		snake_case__	       =   None

		# test with `t5-small`
		snake_case__	       =   TaForConditionalGeneration.from_pretrained(self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		).to(0		)
		snake_case__	       =   model.generate(**UpperCamelCase_		)

		# test with `flan-t5-small`
		snake_case__	       =   TaForConditionalGeneration.from_pretrained(
		    self.dense_act_model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		).to(0		)
		snake_case__	       =   model.generate(**UpperCamelCase_		)
		snake_case__	       =   modules





	def   _snake_case   (		self		)      ->		str:
		import bitsandbytes as bnb

		from transformers import TaForConditionalGeneration

		# test with `t5-small`
		snake_case__	       =   TaForConditionalGeneration.from_pretrained(self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)

		# there was a bug with decoders - this test checks that it is fixed
		self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q      ,       bnb.nn.Linearabit		)		)

		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		).to(0		)
		snake_case__	       =   model.generate(**UpperCamelCase_		)

		# test with `flan-t5-small`
		snake_case__	       =   TaForConditionalGeneration.from_pretrained(
		    self.dense_act_model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		).to(0		)
		snake_case__	       =   model.generate(**UpperCamelCase_		)






class 				__snake_case     (					__magic_name__							):





	def   _snake_case   (		self		)      ->		str:
		super().setUp()
		# model_name
		snake_case__	       =   'bigscience/bloom-560m'
		snake_case__	       =   't5-small'

		# Different types of model

		snake_case__	       =   AutoModel.from_pretrained(self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		# Sequence classification model
		snake_case__	       =   AutoModelForSequenceClassification.from_pretrained(
		    self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		# CausalLM model
		snake_case__	       =   AutoModelForCausalLM.from_pretrained(self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)
		# Seq2seq model
		snake_case__	       =   AutoModelForSeqaSeqLM.from_pretrained(
		    self.seq_to_seq_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='auto'		)





	def   _snake_case   (		self		)      ->		int:
		del self.base_model
		del self.sequence_model
		del self.model_abit
		del self.seq_to_seq_model

		gc.collect()
		torch.cuda.empty_cache()





	def   _snake_case   (		self		)      ->		Optional[int]:
		from bitsandbytes.nn import Paramsabit

		self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit		)

		# Other heads should be nn.Parameter
		self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter		)
		self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter		)
		self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter		)






class 				__snake_case     (					__magic_name__							):





	def   _snake_case   (		self		)      ->		Optional[int]:
		super().setUp()





	def   _snake_case   (		self		)      ->		Optional[int]:
		del self.pipe

		gc.collect()
		torch.cuda.empty_cache()





	def   _snake_case   (		self		)      ->		str:
		snake_case__	       =   pipeline(
		    'text-generation'      ,       model=self.model_name      ,       model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa}      ,       max_new_tokens=self.MAX_NEW_TOKENS      ,       )

		# Real second forward pass
		snake_case__	       =   self.pipe(self.input_text		)
		self.assertIn(pipeline_output[0]['generated_text']      ,       self.EXPECTED_OUTPUTS		)






@require_torch_multi_gpu
class 				__snake_case     (					__magic_name__							):





	def   _snake_case   (		self		)      ->		Optional[Any]:
		super().setUp()





	def   _snake_case   (		self		)      ->		str:
		snake_case__	       =   AutoModelForCausalLM.from_pretrained(
		    self.model_name      ,       load_in_abit=UpperCamelCase_      ,       device_map='balanced'		)

		# Check correct device map
		self.assertEqual(set(model_parallel.hf_device_map.values()		)      ,       {0, 1}		)

		# Check that inference pass works on the model
		snake_case__	       =   self.tokenizer(self.input_text      ,       return_tensors='pt'		)

		# Second real batch
		snake_case__	       =   model_parallel.generate(input_ids=encoded_input['input_ids'].to(0		)      ,       max_new_tokens=10		)
		self.assertIn(self.tokenizer.decode(output_parallel[0]      ,       skip_special_tokens=UpperCamelCase_		)      ,       self.EXPECTED_OUTPUTS		)






class 				__snake_case     (					__magic_name__							):





	def   _snake_case   (		self		)      ->		Optional[int]:
		snake_case__	       =   'facebook/opt-350m'
		super().setUp()





	def   _snake_case   (		self		)      ->		int:
		if version.parse(importlib.metadata.version('bitsandbytes'		)		) < version.parse('0.37.0'		):
			return

		# Step 1: freeze all parameters
		snake_case__	       =   AutoModelForCausalLM.from_pretrained(self.model_name      ,       load_in_abit=UpperCamelCase_		)

		self.assertEqual(set(model.hf_device_map.values()		)      ,       {torch.cuda.current_device()}		)

		for param in model.parameters():
			snake_case__	       =   False  # freeze the model - train adapters later
			if param.ndim == 1:
				# cast the small parameters (e.g. layernorm) to fp32 for stability
				snake_case__	       =   param.data.to(torch.floataa		)

        # Step 2: add adapters
		for _, module in model.named_modules():
			if "OPTAttention" in repr(type(UpperCamelCase_		)		):
				snake_case__	       =   LoRALayer(module.q_proj      ,       rank=16		)
				snake_case__	       =   LoRALayer(module.k_proj      ,       rank=16		)
				snake_case__	       =   LoRALayer(module.v_proj      ,       rank=16		)

        # Step 3: dummy batch
		snake_case__	       =   self.tokenizer('Test batch '      ,       return_tensors='pt'		).to(0		)

		# Step 4: Check if the gradient is not None
		with torch.cuda.amp.autocast():
			snake_case__	       =   model.forward(**UpperCamelCase_		)
			out.logits.norm().backward()

		for module in model.modules():
			if isinstance(UpperCamelCase_      ,       UpperCamelCase_		):
				self.assertTrue(module.adapter[1].weight.grad is not None		)
				self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0		)
			elif isinstance(UpperCamelCase_      ,       nn.Embedding		):
				self.assertTrue(module.weight.grad is None		)






class 				__snake_case     (					__magic_name__							):
	__lowerCAmelCase       						=  '''gpt2-xl'''
	__lowerCAmelCase       						=  3.3191854854152187










'''simple docstring'''




import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


a__					:			Dict          =     logging.get_logger(__name__)

# General docstring
a__					:			Any          =     '''PoolFormerConfig'''

# Base docstring
a__					:			List[str]          =     '''sail/poolformer_s12'''
a__					:			Tuple          =     [1, 512, 7, 7]

# Image classification docstring
a__					:			Dict          =     '''sail/poolformer_s12'''
a__					:			Optional[Any]          =     '''tabby, tabby cat'''

a__					:			Any          =     [
    '''sail/poolformer_s12''',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]




def 						__lowerCamelCase			(	UpperCAmelCase_					,				UpperCAmelCase_ = 0.0					,				UpperCAmelCase_ = False							)   ->Optional[int]:
	if drop_prob == 0.0 or not training:
		return input
	snake_case__	       =   1 - drop_prob
	snake_case__	       =   (input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
	snake_case__	       =   keep_prob + torch.rand(UpperCAmelCase_					,				dtype=input.dtype					,				device=input.device							)
	random_tensor.floor_()  # binarize
	snake_case__	       =   input.div(UpperCAmelCase_							) * random_tensor
	return output






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_ = None		)      ->		None:
		super().__init__()
		snake_case__	       =   drop_prob





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		torch.Tensor:
		return drop_path(UpperCamelCase_      ,       self.drop_prob      ,       self.training		)





	def   _snake_case   (		self		)      ->		str:
		return "p={}".format(self.drop_prob		)






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_=None		)      ->		Optional[Any]:
		super().__init__()
		snake_case__	       =   patch_size if isinstance(UpperCamelCase_      ,       collections.abc.Iterable		) else (patch_size, patch_size)
		snake_case__	       =   stride if isinstance(UpperCamelCase_      ,       collections.abc.Iterable		) else (stride, stride)
		snake_case__	       =   padding if isinstance(UpperCamelCase_      ,       collections.abc.Iterable		) else (padding, padding)

		snake_case__	       =   nn.Convad(UpperCamelCase_      ,       UpperCamelCase_      ,       kernel_size=UpperCamelCase_      ,       stride=UpperCamelCase_      ,       padding=UpperCamelCase_		)
		snake_case__	       =   norm_layer(UpperCamelCase_		) if norm_layer else nn.Identity()





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		int:
		snake_case__	       =   self.projection(UpperCamelCase_		)
		snake_case__	       =   self.norm(UpperCamelCase_		)
		return embeddings






class 				__snake_case     (					nn.GroupNorm							):





	def __init__(		self      ,       UpperCamelCase_      ,       **UpperCamelCase_		)      ->		Dict:
		super().__init__(1      ,       UpperCamelCase_      ,       **UpperCamelCase_		)






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_		)      ->		int:
		super().__init__()
		snake_case__	       =   nn.AvgPoolad(UpperCamelCase_      ,       stride=1      ,       padding=pool_size // 2      ,       count_include_pad=UpperCamelCase_		)





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		str:
		return self.pool(UpperCamelCase_		) - hidden_states






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_		)      ->		int:
		super().__init__()
		snake_case__	       =   nn.Convad(UpperCamelCase_      ,       UpperCamelCase_      ,       1		)
		snake_case__	       =   nn.Convad(UpperCamelCase_      ,       UpperCamelCase_      ,       1		)
		snake_case__	       =   PoolFormerDropPath(UpperCamelCase_		)
		if isinstance(config.hidden_act      ,       UpperCamelCase_		):
			snake_case__	       =   ACTaFN[config.hidden_act]
		else:
			snake_case__	       =   config.hidden_act





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		Any:
		snake_case__	       =   self.conva(UpperCamelCase_		)
		snake_case__	       =   self.act_fn(UpperCamelCase_		)
		snake_case__	       =   self.drop(UpperCamelCase_		)
		snake_case__	       =   self.conva(UpperCamelCase_		)
		snake_case__	       =   self.drop(UpperCamelCase_		)

		return hidden_states






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_		)      ->		Tuple:
		super().__init__()
		snake_case__	       =   PoolFormerPooling(UpperCamelCase_		)
		snake_case__	       =   PoolFormerOutput(UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_		)
		snake_case__	       =   PoolFormerGroupNorm(UpperCamelCase_		)
		snake_case__	       =   PoolFormerGroupNorm(UpperCamelCase_		)

		# Useful for training neural nets
		snake_case__	       =   PoolFormerDropPath(UpperCamelCase_		) if drop_path > 0.0 else nn.Identity()
		snake_case__	       =   config.use_layer_scale
		if config.use_layer_scale:
			snake_case__	       =   nn.Parameter(
			    config.layer_scale_init_value * torch.ones((UpperCamelCase_)		)      ,       requires_grad=UpperCamelCase_		)
			snake_case__	       =   nn.Parameter(
			    config.layer_scale_init_value * torch.ones((UpperCamelCase_)		)      ,       requires_grad=UpperCamelCase_		)





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		List[str]:
		if self.use_layer_scale:
			snake_case__	       =   self.pooling(self.before_norm(UpperCamelCase_		)		)
			snake_case__	       =   self.layer_scale_a.unsqueeze(-1		).unsqueeze(-1		) * pooling_output
			# First residual connection
			snake_case__	       =   hidden_states + self.drop_path(UpperCamelCase_		)
			snake_case__	       =   ()

			snake_case__	       =   self.output(self.after_norm(UpperCamelCase_		)		)
			snake_case__	       =   self.layer_scale_a.unsqueeze(-1		).unsqueeze(-1		) * layer_output
			# Second residual connection
			snake_case__	       =   hidden_states + self.drop_path(UpperCamelCase_		)

			snake_case__	       =   (output,) + outputs
			return outputs

		else:
			snake_case__	       =   self.drop_path(self.pooling(self.before_norm(UpperCamelCase_		)		)		)
			# First residual connection
			snake_case__	       =   pooling_output + hidden_states
			snake_case__	       =   ()

			# Second residual connection inside the PoolFormerOutput block
			snake_case__	       =   self.drop_path(self.output(self.after_norm(UpperCamelCase_		)		)		)
			snake_case__	       =   hidden_states + layer_output

			snake_case__	       =   (output,) + outputs
			return outputs






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_		)      ->		Dict:
		super().__init__()
		snake_case__	       =   config
		# stochastic depth decay rule
		snake_case__	       =   [x.item() for x in torch.linspace(0      ,       config.drop_path_rate      ,       sum(config.depths		)		)]

		# patch embeddings
		snake_case__	       =   []
		for i in range(config.num_encoder_blocks		):
			embeddings.append(
			    PoolFormerEmbeddings(
			        patch_size=config.patch_sizes[i]      ,       stride=config.strides[i]      ,       padding=config.padding[i]      ,       num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]      ,       hidden_size=config.hidden_sizes[i]      ,       )		)
		snake_case__	       =   nn.ModuleList(UpperCamelCase_		)

		# Transformer blocks
		snake_case__	       =   []
		snake_case__	       =   0
		for i in range(config.num_encoder_blocks		):
			# each block consists of layers
			snake_case__	       =   []
			if i != 0:
				cur += config.depths[i - 1]
			for j in range(config.depths[i]		):
				layers.append(
				    PoolFormerLayer(
				        UpperCamelCase_      ,       num_channels=config.hidden_sizes[i]      ,       pool_size=config.pool_size      ,       hidden_size=config.hidden_sizes[i]      ,       intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio		)      ,       drop_path=dpr[cur + j]      ,       )		)
			blocks.append(nn.ModuleList(UpperCamelCase_		)		)

		snake_case__	       =   nn.ModuleList(UpperCamelCase_		)





	def   _snake_case   (		self      ,       UpperCamelCase_      ,       UpperCamelCase_=False      ,       UpperCamelCase_=True		)      ->		Dict:
		snake_case__	       =   () if output_hidden_states else None

		snake_case__	       =   pixel_values
		for idx, layers in enumerate(zip(self.patch_embeddings      ,       self.block		)		):
			snake_case__       ,  snake_case__	       =   layers
			# Get patch embeddings from hidden_states
			snake_case__	       =   embedding_layer(UpperCamelCase_		)
			# Send the embeddings through the blocks
			for _, blk in enumerate(UpperCamelCase_		):
				snake_case__	       =   blk(UpperCamelCase_		)
				snake_case__	       =   layer_outputs[0]

			if output_hidden_states:
				snake_case__	       =   all_hidden_states + (hidden_states,)

		if not return_dict:
			return tuple(v for v in [hidden_states, all_hidden_states] if v is not None		)

		return BaseModelOutputWithNoAttention(last_hidden_state=UpperCamelCase_      ,       hidden_states=UpperCamelCase_		)






class 				__snake_case     (					__magic_name__							):
	__lowerCAmelCase       						=  PoolFormerConfig
	__lowerCAmelCase       						=  '''poolformer'''
	__lowerCAmelCase       						=  '''pixel_values'''
	__lowerCAmelCase       						=  True





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		str:
		if isinstance(UpperCamelCase_      ,       (nn.Linear, nn.Convad)		):
			module.weight.data.normal_(mean=0.0      ,       std=self.config.initializer_range		)
			if module.bias is not None:
				module.bias.data.zero_()
		elif isinstance(UpperCamelCase_      ,       nn.LayerNorm		):
			module.bias.data.zero_()
			module.weight.data.fill_(1.0		)





	def   _snake_case   (		self      ,       UpperCamelCase_      ,       UpperCamelCase_=False		)      ->		int:
		if isinstance(UpperCamelCase_      ,       UpperCamelCase_		):
			snake_case__	       =   value





a__					:			int          =     r'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

a__					:			Union[str, Any]          =     r'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`PoolFormerImageProcessor.__call__`] for details.
'''






@add_start_docstrings(
    '''The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top.'''       ,       __magic_name__       ,       )
class 				__snake_case     (					__magic_name__							):





	def __init__(		self      ,       UpperCamelCase_		)      ->		Any:
		super().__init__(UpperCamelCase_		)
		snake_case__	       =   config

		snake_case__	       =   PoolFormerEncoder(UpperCamelCase_		)

		# Initialize weights and apply final processing
		self.post_init()





	def   _snake_case   (		self		)      ->		str:
		return self.embeddings.patch_embeddings





	@add_start_docstrings_to_model_forward(UpperCamelCase_		)
	@add_code_sample_docstrings(
	    checkpoint=_CHECKPOINT_FOR_DOC      ,       output_type=UpperCamelCase_      ,       config_class=_CONFIG_FOR_DOC      ,       modality='vision'      ,       expected_output=_EXPECTED_OUTPUT_SHAPE      ,       )
	def   _snake_case   (		self      ,       UpperCamelCase_ = None      ,       UpperCamelCase_ = None      ,       UpperCamelCase_ = None      ,       )      ->		Union[Tuple, BaseModelOutputWithNoAttention]:
		snake_case__	       =   (
		    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
		)
		snake_case__	       =   return_dict if return_dict is not None else self.config.use_return_dict

		if pixel_values is None:
			raise ValueError('You have to specify pixel_values'		)

		snake_case__	       =   self.encoder(
		    UpperCamelCase_      ,       output_hidden_states=UpperCamelCase_      ,       return_dict=UpperCamelCase_      ,       )
		snake_case__	       =   encoder_outputs[0]

		if not return_dict:
			return (sequence_output, None) + encoder_outputs[1:]

		return BaseModelOutputWithNoAttention(
		    last_hidden_state=UpperCamelCase_      ,       hidden_states=encoder_outputs.hidden_states      ,       )






class 				__snake_case     (					nn.Module							):





	def __init__(		self      ,       UpperCamelCase_		)      ->		str:
		super().__init__()
		snake_case__	       =   nn.Linear(config.hidden_size      ,       config.hidden_size		)





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		Any:
		snake_case__	       =   self.dense(UpperCamelCase_		)
		return output






@add_start_docstrings(
    '''
    PoolFormer Model transformer with an image classification head on top
    '''       ,       __magic_name__       ,       )
class 				__snake_case     (					__magic_name__							):





	def __init__(		self      ,       UpperCamelCase_		)      ->		Optional[Any]:
		super().__init__(UpperCamelCase_		)
		snake_case__	       =   config.num_labels
		snake_case__	       =   PoolFormerModel(UpperCamelCase_		)

		# Final norm
		snake_case__	       =   PoolFormerGroupNorm(config.hidden_sizes[-1]		)
		# Classifier head
		snake_case__	       =   (
		    nn.Linear(config.hidden_sizes[-1]      ,       config.num_labels		) if config.num_labels > 0 else nn.Identity()
		)

		# Initialize weights and apply final processing
		self.post_init()





	@add_start_docstrings_to_model_forward(UpperCamelCase_		)
	@add_code_sample_docstrings(
	    checkpoint=_IMAGE_CLASS_CHECKPOINT      ,       output_type=UpperCamelCase_      ,       config_class=_CONFIG_FOR_DOC      ,       expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT      ,       )
	def   _snake_case   (		self      ,       UpperCamelCase_ = None      ,       UpperCamelCase_ = None      ,       UpperCamelCase_ = None      ,       UpperCamelCase_ = None      ,       )      ->		Union[Tuple, ImageClassifierOutputWithNoAttention]:
		snake_case__	       =   return_dict if return_dict is not None else self.config.use_return_dict

		snake_case__	       =   self.poolformer(
		    UpperCamelCase_      ,       output_hidden_states=UpperCamelCase_      ,       return_dict=UpperCamelCase_      ,       )

		snake_case__	       =   outputs[0]

		snake_case__	       =   self.classifier(self.norm(UpperCamelCase_		).mean([-2, -1]		)		)

		snake_case__	       =   None
		if labels is not None:
			if self.config.problem_type is None:
				if self.num_labels == 1:
					snake_case__	       =   'regression'
				elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
					snake_case__	       =   'single_label_classification'
				else:
					snake_case__	       =   'multi_label_classification'

			if self.config.problem_type == "regression":
				snake_case__	       =   MSELoss()
				if self.num_labels == 1:
					snake_case__	       =   loss_fct(logits.squeeze()      ,       labels.squeeze()		)
				else:
					snake_case__	       =   loss_fct(UpperCamelCase_      ,       UpperCamelCase_		)
			elif self.config.problem_type == "single_label_classification":
				snake_case__	       =   CrossEntropyLoss()
				snake_case__	       =   loss_fct(logits.view(-1      ,       self.num_labels		)      ,       labels.view(-1		)		)
			elif self.config.problem_type == "multi_label_classification":
				snake_case__	       =   BCEWithLogitsLoss()
				snake_case__	       =   loss_fct(UpperCamelCase_      ,       UpperCamelCase_		)

		if not return_dict:
			snake_case__	       =   (logits,) + outputs[2:]
			return ((loss,) + output) if loss is not None else output

		return ImageClassifierOutputWithNoAttention(loss=UpperCamelCase_      ,       logits=UpperCamelCase_      ,       hidden_states=outputs.hidden_states		)



