






import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


__lowerCamelCase :      Any										=				logging.get_logger(__name__)

# General docstring
__lowerCamelCase :      Dict										=				"""PoolFormerConfig"""

# Base docstring
__lowerCamelCase :      Dict										=				"""sail/poolformer_s12"""
__lowerCamelCase :      Any										=				[1, 512, 7, 7]

# Image classification docstring
__lowerCamelCase :      Optional[int]										=				"""sail/poolformer_s12"""
__lowerCamelCase :      Dict										=				"""tabby, tabby cat"""

__lowerCamelCase :      Dict										=				[
    """sail/poolformer_s12""",
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							str					,  snake_case_ :							float = 0.0					,  snake_case_ :							bool = False       ):
	if drop_prob == 0.0 or not training:
		return input
	snake_case__			:				str       =  1 - drop_prob
	snake_case__			:				Dict       =  (input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
	snake_case__			:				List[Any]       =  keep_prob + torch.rand(snake_case_					,  dtype=input.dtype					,  device=input.device       )
	random_tensor.floor_()  # binarize
	snake_case__			:				Tuple       =  input.div(snake_case_       ) * random_tensor
	return output



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  str							,		__A					:  Optional[float] = None	):
						super().__init__()
						snake_case__			:				Union[str, Any]       =  drop_prob



					def 	_lowercase					( self					:  List[Any]							,		__A					:  torch.Tensor	):
						return drop_path(__A							,		self.drop_prob							,		self.training	)






					def 	_lowercase					( self					:  Tuple	):
						return "p={}".format(self.drop_prob	)



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  Optional[Any]							,		__A					:  List[Any]							,		__A					:  str							,		__A					:  List[Any]							,		__A					:  List[str]							,		__A					:  List[Any]							,		__A					:  Optional[int]=None	):
						super().__init__()
						snake_case__			:				str       =  patch_size if isinstance(__A							,		collections.abc.Iterable	) else (patch_size, patch_size)
						snake_case__			:				int       =  stride if isinstance(__A							,		collections.abc.Iterable	) else (stride, stride)
						snake_case__			:				str       =  padding if isinstance(__A							,		collections.abc.Iterable	) else (padding, padding)

						snake_case__			:				Union[str, Any]       =  nn.Convad(__A							,		__A							,		kernel_size=__A							,		stride=__A							,		padding=__A	)
						snake_case__			:				Tuple       =  norm_layer(__A	) if norm_layer else nn.Identity()






					def 	_lowercase					( self					:  Any							,		__A					:  List[str]	):
						snake_case__			:				str       =  self.projection(__A	)
						snake_case__			:				List[str]       =  self.norm(__A	)
						return embeddings



class 							SCREAMING_SNAKE_CASE__						(    nn.GroupNorm						):





					"""simple docstring"""






					def __init__( self					:  Dict							,		__A					:  Tuple							,		**__A					:  Optional[Any]	):
						super().__init__(1							,		__A							,		**__A	)



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  List[Any]							,		__A					:  List[Any]	):
						super().__init__()
						snake_case__			:				Union[str, Any]       =  nn.AvgPoolad(__A							,		stride=1							,		padding=pool_size // 2							,		count_include_pad=__A	)






					def 	_lowercase					( self					:  Dict							,		__A					:  str	):
						return self.pool(__A	) - hidden_states



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  Optional[int]							,		__A					:  Optional[int]							,		__A					:  str							,		__A					:  Dict							,		__A					:  Dict	):
						super().__init__()
						snake_case__			:				int       =  nn.Convad(__A							,		__A							,		1	)
						snake_case__			:				Optional[Any]       =  nn.Convad(__A							,		__A							,		1	)
						snake_case__			:				Union[str, Any]       =  PoolFormerDropPath(__A	)
						if isinstance(config.hidden_act							,		__A	):
							snake_case__			:				Any       =  ACTaFN[config.hidden_act]
						else:
							snake_case__			:				Union[str, Any]       =  config.hidden_act






					def 	_lowercase					( self					:  Dict							,		__A					:  List[str]	):
						snake_case__			:				str       =  self.conva(__A	)
						snake_case__			:				Tuple       =  self.act_fn(__A	)
						snake_case__			:				Tuple       =  self.drop(__A	)
						snake_case__			:				List[str]       =  self.conva(__A	)
						snake_case__			:				Tuple       =  self.drop(__A	)

						return hidden_states



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  List[Any]							,		__A					:  int							,		__A					:  Tuple							,		__A					:  Tuple							,		__A					:  str							,		__A					:  Dict							,		__A					:  Union[str, Any]	):
						super().__init__()
						snake_case__			:				List[Any]       =  PoolFormerPooling(__A	)
						snake_case__			:				Optional[Any]       =  PoolFormerOutput(__A							,		__A							,		__A							,		__A	)
						snake_case__			:				Optional[Any]       =  PoolFormerGroupNorm(__A	)
						snake_case__			:				int       =  PoolFormerGroupNorm(__A	)

						# Useful for training neural nets
						snake_case__			:				Union[str, Any]       =  PoolFormerDropPath(__A	) if drop_path > 0.0 else nn.Identity()
						snake_case__			:				Any       =  config.use_layer_scale
						if config.use_layer_scale:
							snake_case__			:				Optional[Any]       =  nn.Parameter(
							    config.layer_scale_init_value * torch.ones((__A)	)							,		requires_grad=__A	)
							snake_case__			:				Any       =  nn.Parameter(
							    config.layer_scale_init_value * torch.ones((__A)	)							,		requires_grad=__A	)






					def 	_lowercase					( self					:  Any							,		__A					:  Any	):
						if self.use_layer_scale:
							snake_case__			:				Optional[Any]       =  self.pooling(self.before_norm(__A	)	)
							snake_case__			:				Tuple       =  self.layer_scale_a.unsqueeze(-1	).unsqueeze(-1	) * pooling_output
							# First residual connection
							snake_case__			:				Dict       =  hidden_states + self.drop_path(__A	)
							snake_case__			:				Optional[Any]       =  ()

							snake_case__			:				Optional[Any]       =  self.output(self.after_norm(__A	)	)
							snake_case__			:				Tuple       =  self.layer_scale_a.unsqueeze(-1	).unsqueeze(-1	) * layer_output
							# Second residual connection
							snake_case__			:				int       =  hidden_states + self.drop_path(__A	)

							snake_case__			:				Any       =  (output,) + outputs
							return outputs

						else:
							snake_case__			:				str       =  self.drop_path(self.pooling(self.before_norm(__A	)	)	)
							# First residual connection
							snake_case__			:				Tuple       =  pooling_output + hidden_states
							snake_case__			:				Union[str, Any]       =  ()

							# Second residual connection inside the PoolFormerOutput block
							snake_case__			:				Optional[int]       =  self.drop_path(self.output(self.after_norm(__A	)	)	)
							snake_case__			:				Tuple       =  hidden_states + layer_output

							snake_case__			:				str       =  (output,) + outputs
							return outputs



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  List[Any]							,		__A					:  str	):
						super().__init__()
						snake_case__			:				Optional[int]       =  config
						# stochastic depth decay rule
						snake_case__			:				List[Any]       =  [x.item() for x in torch.linspace(0							,		config.drop_path_rate							,		sum(config.depths	)	)]

						# patch embeddings
						snake_case__			:				Any       =  []
						for i in range(config.num_encoder_blocks	):
							embeddings.append(
							    PoolFormerEmbeddings(
							        patch_size=config.patch_sizes[i]							,		stride=config.strides[i]							,		padding=config.padding[i]							,		num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]							,		hidden_size=config.hidden_sizes[i]							,		)	)
						snake_case__			:				List[str]       =  nn.ModuleList(__A	)

						# Transformer blocks
						snake_case__			:				Dict       =  []
						snake_case__			:				List[str]       =  0
						for i in range(config.num_encoder_blocks	):
							# each block consists of layers
							snake_case__			:				List[str]       =  []
							if i != 0:
								cur += config.depths[i - 1]
							for j in range(config.depths[i]	):
								layers.append(
								    PoolFormerLayer(
								        __A							,		num_channels=config.hidden_sizes[i]							,		pool_size=config.pool_size							,		hidden_size=config.hidden_sizes[i]							,		intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio	)							,		drop_path=dpr[cur + j]							,		)	)
							blocks.append(nn.ModuleList(__A	)	)

						snake_case__			:				Optional[int]       =  nn.ModuleList(__A	)






					def 	_lowercase					( self					:  Optional[Any]							,		__A					:  Tuple							,		__A					:  List[str]=False							,		__A					:  int=True	):
						snake_case__			:				Dict       =  () if output_hidden_states else None

						snake_case__			:				str       =  pixel_values
						for idx, layers in enumerate(zip(self.patch_embeddings							,		self.block	)	):
							snake_case__,						snake_case__			:				Tuple       =  layers
							# Get patch embeddings from hidden_states
							snake_case__			:				str       =  embedding_layer(__A	)
							# Send the embeddings through the blocks
							for _, blk in enumerate(__A	):
								snake_case__			:				int       =  blk(__A	)
								snake_case__			:				List[Any]       =  layer_outputs[0]

							if output_hidden_states:
								snake_case__			:				Optional[int]       =  all_hidden_states + (hidden_states,)

						if not return_dict:
							return tuple(v for v in [hidden_states, all_hidden_states] if v is not None	)

						return BaseModelOutputWithNoAttention(last_hidden_state=__A							,		hidden_states=__A	)



class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_						):





					"""simple docstring"""






					a_     =		PoolFormerConfig
					a_     =		"poolformer"
					a_     =		"pixel_values"
					a_     =		True



					def 	_lowercase					( self					:  Dict							,		__A					:  Any	):
						if isinstance(__A							,		(nn.Linear, nn.Convad)	):
							module.weight.data.normal_(mean=0.0							,		std=self.config.initializer_range	)
							if module.bias is not None:
								module.bias.data.zero_()
						elif isinstance(__A							,		nn.LayerNorm	):
							module.bias.data.zero_()
							module.weight.data.fill_(1.0	)






					def 	_lowercase					( self					:  Tuple							,		__A					:  List[Any]							,		__A					:  Dict=False	):
						if isinstance(__A							,		__A	):
							snake_case__			:				str       =  value
__lowerCamelCase :      Optional[int]										=				R"""
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""

__lowerCamelCase :      Dict										=				R"""
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`PoolFormerImageProcessor.__call__`] for details.
"""



@add_start_docstrings(
    "The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top."			,       UpperCamelCase_			,       )
class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_						):





					"""simple docstring"""






					def __init__( self					:  str							,		__A					:  Optional[int]	):
						super().__init__(__A	)
						snake_case__			:				str       =  config

						snake_case__			:				Union[str, Any]       =  PoolFormerEncoder(__A	)

						# Initialize weights and apply final processing
						self.post_init()



					def 	_lowercase					( self					:  Union[str, Any]	):
						return self.embeddings.patch_embeddings






					@add_start_docstrings_to_model_forward(__A	)
					@add_code_sample_docstrings(
					    checkpoint=_CHECKPOINT_FOR_DOC							,		output_type=__A							,		config_class=_CONFIG_FOR_DOC							,		modality="vision"							,		expected_output=_EXPECTED_OUTPUT_SHAPE							,		)
					def 	_lowercase					( self					:  Any							,		__A					:  Optional[torch.FloatTensor] = None							,		__A					:  Optional[bool] = None							,		__A					:  Optional[bool] = None							,		):
						snake_case__			:				List[str]       =  (
						    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
						)
						snake_case__			:				Tuple       =  return_dict if return_dict is not None else self.config.use_return_dict

						if pixel_values is None:
							raise ValueError("You have to specify pixel_values"	)

						snake_case__			:				Any       =  self.encoder(
						    __A							,		output_hidden_states=__A							,		return_dict=__A							,		)
						snake_case__			:				List[str]       =  encoder_outputs[0]

						if not return_dict:
							return (sequence_output, None) + encoder_outputs[1:]

						return BaseModelOutputWithNoAttention(
						    last_hidden_state=__A							,		hidden_states=encoder_outputs.hidden_states							,		)



class 							SCREAMING_SNAKE_CASE__						(    nn.Module						):





					"""simple docstring"""






					def __init__( self					:  List[Any]							,		__A					:  Optional[Any]	):
						super().__init__()
						snake_case__			:				Dict       =  nn.Linear(config.hidden_size							,		config.hidden_size	)






					def 	_lowercase					( self					:  Dict							,		__A					:  List[Any]	):
						snake_case__			:				int       =  self.dense(__A	)
						return output



@add_start_docstrings(
    "\n    PoolFormer Model transformer with an image classification head on top\n    "			,       UpperCamelCase_			,       )
class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_						):





					"""simple docstring"""






					def __init__( self					:  Any							,		__A					:  Tuple	):
						super().__init__(__A	)
						snake_case__			:				Optional[Any]       =  config.num_labels
						snake_case__			:				int       =  PoolFormerModel(__A	)

						# Final norm
						snake_case__			:				Any       =  PoolFormerGroupNorm(config.hidden_sizes[-1]	)
						# Classifier head
						snake_case__			:				int       =  (
						    nn.Linear(config.hidden_sizes[-1]							,		config.num_labels	) if config.num_labels > 0 else nn.Identity()
						)

						# Initialize weights and apply final processing
						self.post_init()






					@add_start_docstrings_to_model_forward(__A	)
					@add_code_sample_docstrings(
					    checkpoint=_IMAGE_CLASS_CHECKPOINT							,		output_type=__A							,		config_class=_CONFIG_FOR_DOC							,		expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT							,		)
					def 	_lowercase					( self					:  int							,		__A					:  Optional[torch.FloatTensor] = None							,		__A					:  Optional[torch.LongTensor] = None							,		__A					:  Optional[bool] = None							,		__A					:  Optional[bool] = None							,		):
						snake_case__			:				str       =  return_dict if return_dict is not None else self.config.use_return_dict

						snake_case__			:				List[str]       =  self.poolformer(
						    __A							,		output_hidden_states=__A							,		return_dict=__A							,		)

						snake_case__			:				Optional[Any]       =  outputs[0]

						snake_case__			:				str       =  self.classifier(self.norm(__A	).mean([-2, -1]	)	)

						snake_case__			:				Any       =  None
						if labels is not None:
							if self.config.problem_type is None:
								if self.num_labels == 1:
									snake_case__			:				List[str]       =  "regression"
								elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
									snake_case__			:				str       =  "single_label_classification"
								else:
									snake_case__			:				Any       =  "multi_label_classification"

							if self.config.problem_type == "regression":
								snake_case__			:				Tuple       =  MSELoss()
								if self.num_labels == 1:
									snake_case__			:				Optional[Any]       =  loss_fct(logits.squeeze()							,		labels.squeeze()	)
								else:
									snake_case__			:				List[Any]       =  loss_fct(__A							,		__A	)
							elif self.config.problem_type == "single_label_classification":
								snake_case__			:				Optional[int]       =  CrossEntropyLoss()
								snake_case__			:				Any       =  loss_fct(logits.view(-1							,		self.num_labels	)							,		labels.view(-1	)	)
							elif self.config.problem_type == "multi_label_classification":
								snake_case__			:				int       =  BCEWithLogitsLoss()
								snake_case__			:				Union[str, Any]       =  loss_fct(__A							,		__A	)

						if not return_dict:
							snake_case__			:				str       =  (logits,) + outputs[2:]
							return ((loss,) + output) if loss is not None else output

						return ImageClassifierOutputWithNoAttention(loss=__A							,		logits=__A							,		hidden_states=outputs.hidden_states	)











import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		PhobertTokenizer
					a_     =		False



					def 	_lowercase					( self					:  List[str]	):
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						snake_case__			:				Optional[int]       =  ["T@@", "i", "I", "R@@", "r", "e@@"]
						snake_case__			:				int       =  dict(zip(__A							,		range(len(__A	)	)	)	)
						snake_case__			:				Dict       =  ["#version: 0.2", "l à</w>"]
						snake_case__			:				Optional[Any]       =  {"unk_token": "<unk>"}

						snake_case__			:				Tuple       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["vocab_file"]	)
						snake_case__			:				Union[str, Any]       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["merges_file"]	)

						with open(self.vocab_file							,		"w"							,		encoding="utf-8"	) as fp:
							for token in vocab_tokens:
								fp.write(f'''{token} {vocab_tokens[token]}\n'''	)
						with open(self.merges_file							,		"w"							,		encoding="utf-8"	) as fp:
							fp.write("\n".join(__A	)	)



					def 	_lowercase					( self					:  List[str]							,		**__A					:  Any	):
						kwargs.update(self.special_tokens_map	)
						return PhobertTokenizer.from_pretrained(self.tmpdirname							,		**__A	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  List[Any]	):
						snake_case__			:				str       =  "Tôi là VinAI Research"
						snake_case__			:				int       =  "T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
						return input_text, output_text






					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				int       =  PhobertTokenizer(self.vocab_file							,		self.merges_file							,		**self.special_tokens_map	)
						snake_case__			:				Tuple       =  "Tôi là VinAI Research"
						snake_case__			:				List[Any]       =  "T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
						snake_case__			:				int       =  tokenizer.tokenize(__A	)
						print(__A	)
						self.assertListEqual(__A							,		__A	)

						snake_case__			:				Any       =  tokens + [tokenizer.unk_token]

						snake_case__			:				Any       =  [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		__A	)



