





import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


UpperCAmelCase__						:			Tuple									=logging.get_logger(__name__)

# General docstring
UpperCAmelCase__						:			Tuple									='''PoolFormerConfig'''

# Base docstring
UpperCAmelCase__						:			str									='''sail/poolformer_s12'''
UpperCAmelCase__						:			str									=[1, 5_12, 7, 7]

# Image classification docstring
UpperCAmelCase__						:			Optional[int]									='''sail/poolformer_s12'''
UpperCAmelCase__						:			Any									='''tabby, tabby cat'''

UpperCAmelCase__						:			Optional[int]									=[
    '''sail/poolformer_s12''',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]





def      _lowercase					(						_UpperCAmelCase			,      _UpperCAmelCase = 0.0			,      _UpperCAmelCase = False				)						->  str:
				if drop_prob == 0.0 or not training:
								return input
				lowerCamelCase	 =1 - drop_prob
				lowerCamelCase	 =(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
				lowerCamelCase	 =keep_prob + torch.rand(_UpperCAmelCase			,      dtype=input.dtype			,      device=input.device				)
				random_tensor.floor_()  # binarize
				lowerCamelCase	 =input.div(_UpperCAmelCase				) * random_tensor
				return output

class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_ = None							):
								super().__init__()
								lowerCamelCase	 =drop_prob


				def _snake_case       (		self   , UpperCAmelCase_							):
								return drop_path(UpperCAmelCase_   , self.drop_prob   , self.training							)


				def _snake_case       (		self							):
								return "p={}".format(self.drop_prob							)




class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_=None							):
								super().__init__()
								lowerCamelCase	 =patch_size if isinstance(UpperCAmelCase_   , collections.abc.Iterable							) else (patch_size, patch_size)
								lowerCamelCase	 =stride if isinstance(UpperCAmelCase_   , collections.abc.Iterable							) else (stride, stride)
								lowerCamelCase	 =padding if isinstance(UpperCAmelCase_   , collections.abc.Iterable							) else (padding, padding)

								lowerCamelCase	 =nn.Convad(UpperCAmelCase_   , UpperCAmelCase_   , kernel_size=UpperCAmelCase_   , stride=UpperCAmelCase_   , padding=UpperCAmelCase_							)
								lowerCamelCase	 =norm_layer(UpperCAmelCase_							) if norm_layer else nn.Identity()


				def _snake_case       (		self   , UpperCAmelCase_							):
								lowerCamelCase	 =self.projection(UpperCAmelCase_							)
								lowerCamelCase	 =self.norm(UpperCAmelCase_							)
								return embeddings




class       __A     (						nn.GroupNorm    ):


				def __init__(		self   , UpperCAmelCase_   , **UpperCAmelCase_							):
								super().__init__(1   , UpperCAmelCase_   , **UpperCAmelCase_							)




class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__()
								lowerCamelCase	 =nn.AvgPoolad(UpperCAmelCase_   , stride=1   , padding=pool_size // 2   , count_include_pad=UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_							):
								return self.pool(UpperCAmelCase_							) - hidden_states




class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							):
								super().__init__()
								lowerCamelCase	 =nn.Convad(UpperCAmelCase_   , UpperCAmelCase_   , 1							)
								lowerCamelCase	 =nn.Convad(UpperCAmelCase_   , UpperCAmelCase_   , 1							)
								lowerCamelCase	 =PoolFormerDropPath(UpperCAmelCase_							)
								if isinstance(config.hidden_act   , UpperCAmelCase_							):
												lowerCamelCase	 =ACTaFN[config.hidden_act]
								else:
												lowerCamelCase	 =config.hidden_act


				def _snake_case       (		self   , UpperCAmelCase_							):
								lowerCamelCase	 =self.conva(UpperCAmelCase_							)
								lowerCamelCase	 =self.act_fn(UpperCAmelCase_							)
								lowerCamelCase	 =self.drop(UpperCAmelCase_							)
								lowerCamelCase	 =self.conva(UpperCAmelCase_							)
								lowerCamelCase	 =self.drop(UpperCAmelCase_							)

								return hidden_states




class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							):
								super().__init__()
								lowerCamelCase	 =PoolFormerPooling(UpperCAmelCase_							)
								lowerCamelCase	 =PoolFormerOutput(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							)
								lowerCamelCase	 =PoolFormerGroupNorm(UpperCAmelCase_							)
								lowerCamelCase	 =PoolFormerGroupNorm(UpperCAmelCase_							)

								# Useful for training neural nets
								lowerCamelCase	 =PoolFormerDropPath(UpperCAmelCase_							) if drop_path > 0.0 else nn.Identity()
								lowerCamelCase	 =config.use_layer_scale
								if config.use_layer_scale:
												lowerCamelCase	 =nn.Parameter(
												    config.layer_scale_init_value * torch.ones((UpperCAmelCase_)							)   , requires_grad=UpperCAmelCase_							)
												lowerCamelCase	 =nn.Parameter(
												    config.layer_scale_init_value * torch.ones((UpperCAmelCase_)							)   , requires_grad=UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_							):
								if self.use_layer_scale:
												lowerCamelCase	 =self.pooling(self.before_norm(UpperCAmelCase_							)							)
												lowerCamelCase	 =self.layer_scale_a.unsqueeze(-1							).unsqueeze(-1							) * pooling_output
												# First residual connection
												lowerCamelCase	 =hidden_states + self.drop_path(UpperCAmelCase_							)
												lowerCamelCase	 =()

												lowerCamelCase	 =self.output(self.after_norm(UpperCAmelCase_							)							)
												lowerCamelCase	 =self.layer_scale_a.unsqueeze(-1							).unsqueeze(-1							) * layer_output
												# Second residual connection
												lowerCamelCase	 =hidden_states + self.drop_path(UpperCAmelCase_							)

												lowerCamelCase	 =(output,) + outputs
												return outputs

								else:
												lowerCamelCase	 =self.drop_path(self.pooling(self.before_norm(UpperCAmelCase_							)							)							)
												# First residual connection
												lowerCamelCase	 =pooling_output + hidden_states
												lowerCamelCase	 =()

												# Second residual connection inside the PoolFormerOutput block
												lowerCamelCase	 =self.drop_path(self.output(self.after_norm(UpperCAmelCase_							)							)							)
												lowerCamelCase	 =hidden_states + layer_output

												lowerCamelCase	 =(output,) + outputs
												return outputs




class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__()
								lowerCamelCase	 =config
								# stochastic depth decay rule
								lowerCamelCase	 =[x.item() for x in torch.linspace(0   , config.drop_path_rate   , sum(config.depths							)							)]

								# patch embeddings
								lowerCamelCase	 =[]
								for i in range(config.num_encoder_blocks							):
												embeddings.append(
												    PoolFormerEmbeddings(
												        patch_size=config.patch_sizes[i]   , stride=config.strides[i]   , padding=config.padding[i]   , num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]   , hidden_size=config.hidden_sizes[i]   , )							)
								lowerCamelCase	 =nn.ModuleList(UpperCAmelCase_							)

								# Transformer blocks
								lowerCamelCase	 =[]
								lowerCamelCase	 =0
								for i in range(config.num_encoder_blocks							):
												# each block consists of layers
												lowerCamelCase	 =[]
												if i != 0:
																cur += config.depths[i - 1]
												for j in range(config.depths[i]							):
																layers.append(
																    PoolFormerLayer(
																        UpperCAmelCase_   , num_channels=config.hidden_sizes[i]   , pool_size=config.pool_size   , hidden_size=config.hidden_sizes[i]   , intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio							)   , drop_path=dpr[cur + j]   , )							)
												blocks.append(nn.ModuleList(UpperCAmelCase_							)							)

								lowerCamelCase	 =nn.ModuleList(UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_=False   , UpperCAmelCase_=True							):
								lowerCamelCase	 =() if output_hidden_states else None

								lowerCamelCase	 =pixel_values
								for idx, layers in enumerate(zip(self.patch_embeddings   , self.block							)							):
												lowerCamelCase ,  lowerCamelCase	 =layers
												# Get patch embeddings from hidden_states
												lowerCamelCase	 =embedding_layer(UpperCAmelCase_							)
												# Send the embeddings through the blocks
												for _, blk in enumerate(UpperCAmelCase_							):
																lowerCamelCase	 =blk(UpperCAmelCase_							)
																lowerCamelCase	 =layer_outputs[0]

												if output_hidden_states:
																lowerCamelCase	 =all_hidden_states + (hidden_states,)

								if not return_dict:
												return tuple(v for v in [hidden_states, all_hidden_states] if v is not None							)

								return BaseModelOutputWithNoAttention(last_hidden_state=UpperCAmelCase_   , hidden_states=UpperCAmelCase_							)




class       __A     (						a    ):
				__A      =				PoolFormerConfig
				__A      =				"""poolformer"""
				__A      =				"""pixel_values"""
				__A      =				True


				def _snake_case       (		self   , UpperCAmelCase_							):
								if isinstance(UpperCAmelCase_   , (nn.Linear, nn.Convad)							):
												module.weight.data.normal_(mean=0.0   , std=self.config.initializer_range							)
												if module.bias is not None:
																module.bias.data.zero_()
								elif isinstance(UpperCAmelCase_   , nn.LayerNorm							):
												module.bias.data.zero_()
												module.weight.data.fill_(1.0							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_=False							):
								if isinstance(UpperCAmelCase_   , UpperCAmelCase_							):
												lowerCamelCase	 =value




UpperCAmelCase__						:			Dict									=r'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

UpperCAmelCase__						:			Optional[Any]									=r'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`PoolFormerImageProcessor.__call__`] for details.
'''

@add_start_docstrings(
    """The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top."""    ,					a    ,					)
class       __A     (						a    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__(UpperCAmelCase_							)
								lowerCamelCase	 =config

								lowerCamelCase	 =PoolFormerEncoder(UpperCAmelCase_							)

								# Initialize weights and apply final processing
								self.post_init()


				def _snake_case       (		self							):
								return self.embeddings.patch_embeddings


				@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
				@add_code_sample_docstrings(
				    checkpoint=_CHECKPOINT_FOR_DOC   , output_type=UpperCAmelCase_   , config_class=_CONFIG_FOR_DOC   , modality="""vision"""   , expected_output=_EXPECTED_OUTPUT_SHAPE   , )
				def _snake_case       (		self   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , ):
								lowerCamelCase	 =(
								    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
								)
								lowerCamelCase	 =return_dict if return_dict is not None else self.config.use_return_dict

								if pixel_values is None:
												raise ValueError("""You have to specify pixel_values"""							)

								lowerCamelCase	 =self.encoder(
								    UpperCAmelCase_   , output_hidden_states=UpperCAmelCase_   , return_dict=UpperCAmelCase_   , )
								lowerCamelCase	 =encoder_outputs[0]

								if not return_dict:
												return (sequence_output, None) + encoder_outputs[1:]

								return BaseModelOutputWithNoAttention(
								    last_hidden_state=UpperCAmelCase_   , hidden_states=encoder_outputs.hidden_states   , )




class       __A     (						nn.Module    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__()
								lowerCamelCase	 =nn.Linear(config.hidden_size   , config.hidden_size							)


				def _snake_case       (		self   , UpperCAmelCase_							):
								lowerCamelCase	 =self.dense(UpperCAmelCase_							)
								return output




@add_start_docstrings(
    """
    PoolFormer Model transformer with an image classification head on top
    """    ,					a    ,					)
class       __A     (						a    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__(UpperCAmelCase_							)
								lowerCamelCase	 =config.num_labels
								lowerCamelCase	 =PoolFormerModel(UpperCAmelCase_							)

								# Final norm
								lowerCamelCase	 =PoolFormerGroupNorm(config.hidden_sizes[-1]							)
								# Classifier head
								lowerCamelCase	 =(
								    nn.Linear(config.hidden_sizes[-1]   , config.num_labels							) if config.num_labels > 0 else nn.Identity()
								)

								# Initialize weights and apply final processing
								self.post_init()


				@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
				@add_code_sample_docstrings(
				    checkpoint=_IMAGE_CLASS_CHECKPOINT   , output_type=UpperCAmelCase_   , config_class=_CONFIG_FOR_DOC   , expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT   , )
				def _snake_case       (		self   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , ):
								lowerCamelCase	 =return_dict if return_dict is not None else self.config.use_return_dict

								lowerCamelCase	 =self.poolformer(
								    UpperCAmelCase_   , output_hidden_states=UpperCAmelCase_   , return_dict=UpperCAmelCase_   , )

								lowerCamelCase	 =outputs[0]

								lowerCamelCase	 =self.classifier(self.norm(UpperCAmelCase_							).mean([-2, -1]							)							)

								lowerCamelCase	 =None
								if labels is not None:
												if self.config.problem_type is None:
																if self.num_labels == 1:
																				lowerCamelCase	 ="""regression"""
																elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																				lowerCamelCase	 ="""single_label_classification"""
																else:
																				lowerCamelCase	 ="""multi_label_classification"""

												if self.config.problem_type == "regression":
																lowerCamelCase	 =MSELoss()
																if self.num_labels == 1:
																				lowerCamelCase	 =loss_fct(logits.squeeze()   , labels.squeeze()							)
																else:
																				lowerCamelCase	 =loss_fct(UpperCAmelCase_   , UpperCAmelCase_							)
												elif self.config.problem_type == "single_label_classification":
																lowerCamelCase	 =CrossEntropyLoss()
																lowerCamelCase	 =loss_fct(logits.view(-1   , self.num_labels							)   , labels.view(-1							)							)
												elif self.config.problem_type == "multi_label_classification":
																lowerCamelCase	 =BCEWithLogitsLoss()
																lowerCamelCase	 =loss_fct(UpperCAmelCase_   , UpperCAmelCase_							)

								if not return_dict:
												lowerCamelCase	 =(logits,) + outputs[2:]
												return ((loss,) + output) if loss is not None else output

								return ImageClassifierOutputWithNoAttention(loss=UpperCAmelCase_   , logits=UpperCAmelCase_   , hidden_states=outputs.hidden_states							)












import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase__						:			Optional[Any]									=logging.get_logger(__name__)

UpperCAmelCase__						:			Union[str, Any]									={'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__						:			str									={
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__						:			Union[str, Any]									={
    '''Salesforce/codegen-350M-mono''': 20_48,
}

class       __A     (						a    ):
				__A      =				VOCAB_FILES_NAMES
				__A      =				PRETRAINED_VOCAB_FILES_MAP
				__A      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A      =				["""input_ids""", """attention_mask"""]
				__A      =				CodeGenTokenizer


				def __init__(		self   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_=False   , **UpperCAmelCase_   , ):
								super().__init__(
								    UpperCAmelCase_   , UpperCAmelCase_   , tokenizer_file=UpperCAmelCase_   , unk_token=UpperCAmelCase_   , bos_token=UpperCAmelCase_   , eos_token=UpperCAmelCase_   , add_prefix_space=UpperCAmelCase_   , **UpperCAmelCase_   , )

								if kwargs.pop("""add_bos_token"""   , UpperCAmelCase_							):
												lowerCamelCase	 =kwargs.pop("""name_or_path"""   , """"""							)
												raise ValueError(
												    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
												    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
												    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
												    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
												    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
												    """ so that the fast tokenizer works correctly."""							)

								lowerCamelCase	 =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
								if pre_tok_state.get("""add_prefix_space"""   , UpperCAmelCase_							) != add_prefix_space:
												lowerCamelCase	 =getattr(UpperCAmelCase_   , pre_tok_state.pop("""type"""							)							)
												lowerCamelCase	 =add_prefix_space
												lowerCamelCase	 =pre_tok_class(**UpperCAmelCase_							)

								lowerCamelCase	 =add_prefix_space


				def _snake_case       (		self   , *UpperCAmelCase_   , **UpperCAmelCase_							):
								lowerCamelCase	 =kwargs.get("""is_split_into_words"""   , UpperCAmelCase_							)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*UpperCAmelCase_   , **UpperCAmelCase_							)


				def _snake_case       (		self   , *UpperCAmelCase_   , **UpperCAmelCase_							):
								lowerCamelCase	 =kwargs.get("""is_split_into_words"""   , UpperCAmelCase_							)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*UpperCAmelCase_   , **UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = None							):
								lowerCamelCase	 =self._tokenizer.model.save(UpperCAmelCase_   , name=UpperCAmelCase_							)
								return tuple(UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = False   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , **UpperCAmelCase_   , ):
								lowerCamelCase	 =super().decode(
								    token_ids=UpperCAmelCase_   , skip_special_tokens=UpperCAmelCase_   , clean_up_tokenization_spaces=UpperCAmelCase_   , **UpperCAmelCase_   , )

								if truncate_before_pattern is not None and len(UpperCAmelCase_							) > 0:
												lowerCamelCase	 =self.truncate(UpperCAmelCase_   , UpperCAmelCase_							)

								return decoded_text


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_							):
								def find_re(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							):
												lowerCamelCase	 =pattern.search(UpperCAmelCase_   , UpperCAmelCase_							)
												return m.start() if m else -1

								lowerCamelCase	 =[re.compile(UpperCAmelCase_   , re.MULTILINE							) for pattern in truncate_before_pattern]

								lowerCamelCase	 =list(re.finditer("""^print"""   , UpperCAmelCase_   , re.MULTILINE							)							)

								if len(UpperCAmelCase_							) > 1:
												lowerCamelCase	 =completion[: prints[1].start()]

								lowerCamelCase	 =list(re.finditer("""^def"""   , UpperCAmelCase_   , re.MULTILINE							)							)

								if len(UpperCAmelCase_							) > 1:
												lowerCamelCase	 =completion[: defs[1].start()]

								lowerCamelCase	 =0

								lowerCamelCase	 =[
								    pos for pos in [find_re(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							) for terminal in terminals] if pos != -1
								]

								if len(UpperCAmelCase_							) > 0:
												return completion[: min(UpperCAmelCase_							)]
								else:
												return completion





