





import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


UpperCamelCase__											= logging.get_logger(__name__)

# General docstring
UpperCamelCase__											= "PoolFormerConfig"

# Base docstring
UpperCamelCase__											= "sail/poolformer_s12"
UpperCamelCase__											= [1, 512, 7, 7]

# Image classification docstring
UpperCamelCase__											= "sail/poolformer_s12"
UpperCamelCase__											= "tabby, tabby cat"

UpperCamelCase__											= [
    "sail/poolformer_s12",
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]





def     _UpperCamelCase     (a__   :Optional[int]   ,			a__   :float = 0.0   ,			a__   :bool = False       ):






							"""simple docstring"""





							if drop_prob == 0.0 or not training:
														return input
							UpperCamelCase__				     =			1 - drop_prob
							UpperCamelCase__				     =			(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
							UpperCamelCase__				     =			keep_prob + torch.rand(a__   ,			dtype=input.dtype   ,			device=input.device       )
							random_tensor.floor_()  # binarize
							UpperCamelCase__				     =			input.div(a__       ) * random_tensor
							return output

class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase = None  ):
														super().__init__()
														UpperCamelCase__				     =			drop_prob






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return drop_path(__lowerCAmelCase  , self.drop_prob  , self.training  )







							def 				_lowerCamelCase		(						self  ):
														return "p={}".format(self.drop_prob  )






class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase=None  ):
														super().__init__()
														UpperCamelCase__				     =			patch_size if isinstance(__lowerCAmelCase  , collections.abc.Iterable  ) else (patch_size, patch_size)
														UpperCamelCase__				     =			stride if isinstance(__lowerCAmelCase  , collections.abc.Iterable  ) else (stride, stride)
														UpperCamelCase__				     =			padding if isinstance(__lowerCAmelCase  , collections.abc.Iterable  ) else (padding, padding)

														UpperCamelCase__				     =			nn.Convad(__lowerCAmelCase  , __lowerCAmelCase  , kernel_size=__lowerCAmelCase  , stride=__lowerCAmelCase  , padding=__lowerCAmelCase  )
														UpperCamelCase__				     =			norm_layer(__lowerCAmelCase  ) if norm_layer else nn.Identity()







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.projection(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.norm(__lowerCAmelCase  )
														return embeddings






class        __SCREAMING_SNAKE_CASE					(     nn.GroupNorm       ):






							def __init__(						self  , __lowerCAmelCase  , **__lowerCAmelCase  ):
														super().__init__(1  , __lowerCAmelCase  , **__lowerCAmelCase  )






class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase  ):
														super().__init__()
														UpperCamelCase__				     =			nn.AvgPoolad(__lowerCAmelCase  , stride=1  , padding=pool_size // 2  , count_include_pad=__lowerCAmelCase  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.pool(__lowerCAmelCase  ) - hidden_states






class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														super().__init__()
														UpperCamelCase__				     =			nn.Convad(__lowerCAmelCase  , __lowerCAmelCase  , 1  )
														UpperCamelCase__				     =			nn.Convad(__lowerCAmelCase  , __lowerCAmelCase  , 1  )
														UpperCamelCase__				     =			PoolFormerDropPath(__lowerCAmelCase  )
														if isinstance(config.hidden_act  , __lowerCAmelCase  ):
																					UpperCamelCase__				     =			ACTaFN[config.hidden_act]
														else:
																					UpperCamelCase__				     =			config.hidden_act







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.conva(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.act_fn(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.drop(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.conva(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.drop(__lowerCAmelCase  )

														return hidden_states






class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														super().__init__()
														UpperCamelCase__				     =			PoolFormerPooling(__lowerCAmelCase  )
														UpperCamelCase__				     =			PoolFormerOutput(__lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
														UpperCamelCase__				     =			PoolFormerGroupNorm(__lowerCAmelCase  )
														UpperCamelCase__				     =			PoolFormerGroupNorm(__lowerCAmelCase  )

														# Useful for training neural nets
														UpperCamelCase__				     =			PoolFormerDropPath(__lowerCAmelCase  ) if drop_path > 0.0 else nn.Identity()
														UpperCamelCase__				     =			config.use_layer_scale
														if config.use_layer_scale:
																					UpperCamelCase__				     =			nn.Parameter(
																					    config.layer_scale_init_value * torch.ones((__lowerCAmelCase)  )  , requires_grad=__lowerCAmelCase  )
																					UpperCamelCase__				     =			nn.Parameter(
																					    config.layer_scale_init_value * torch.ones((__lowerCAmelCase)  )  , requires_grad=__lowerCAmelCase  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if self.use_layer_scale:
																					UpperCamelCase__				     =			self.pooling(self.before_norm(__lowerCAmelCase  )  )
																					UpperCamelCase__				     =			self.layer_scale_a.unsqueeze(-1  ).unsqueeze(-1  ) * pooling_output
																					# First residual connection
																					UpperCamelCase__				     =			hidden_states + self.drop_path(__lowerCAmelCase  )
																					UpperCamelCase__				     =			()

																					UpperCamelCase__				     =			self.output(self.after_norm(__lowerCAmelCase  )  )
																					UpperCamelCase__				     =			self.layer_scale_a.unsqueeze(-1  ).unsqueeze(-1  ) * layer_output
																					# Second residual connection
																					UpperCamelCase__				     =			hidden_states + self.drop_path(__lowerCAmelCase  )

																					UpperCamelCase__				     =			(output,) + outputs
																					return outputs

														else:
																					UpperCamelCase__				     =			self.drop_path(self.pooling(self.before_norm(__lowerCAmelCase  )  )  )
																					# First residual connection
																					UpperCamelCase__				     =			pooling_output + hidden_states
																					UpperCamelCase__				     =			()

																					# Second residual connection inside the PoolFormerOutput block
																					UpperCamelCase__				     =			self.drop_path(self.output(self.after_norm(__lowerCAmelCase  )  )  )
																					UpperCamelCase__				     =			hidden_states + layer_output

																					UpperCamelCase__				     =			(output,) + outputs
																					return outputs






class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase  ):
														super().__init__()
														UpperCamelCase__				     =			config
														# stochastic depth decay rule
														UpperCamelCase__				     =			[x.item() for x in torch.linspace(0  , config.drop_path_rate  , sum(config.depths  )  )]

														# patch embeddings
														UpperCamelCase__				     =			[]
														for i in range(config.num_encoder_blocks  ):
																					embeddings.append(
																					    PoolFormerEmbeddings(
																					        patch_size=config.patch_sizes[i]  , stride=config.strides[i]  , padding=config.padding[i]  , num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]  , hidden_size=config.hidden_sizes[i]  , )  )
														UpperCamelCase__				     =			nn.ModuleList(__lowerCAmelCase  )

														# Transformer blocks
														UpperCamelCase__				     =			[]
														UpperCamelCase__				     =			0
														for i in range(config.num_encoder_blocks  ):
																					# each block consists of layers
																					UpperCamelCase__				     =			[]
																					if i != 0:
																												cur += config.depths[i - 1]
																					for j in range(config.depths[i]  ):
																												layers.append(
																												    PoolFormerLayer(
																												        __lowerCAmelCase  , num_channels=config.hidden_sizes[i]  , pool_size=config.pool_size  , hidden_size=config.hidden_sizes[i]  , intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio  )  , drop_path=dpr[cur + j]  , )  )
																					blocks.append(nn.ModuleList(__lowerCAmelCase  )  )

														UpperCamelCase__				     =			nn.ModuleList(__lowerCAmelCase  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase=False  , __lowerCAmelCase=True  ):
														UpperCamelCase__				     =			() if output_hidden_states else None

														UpperCamelCase__				     =			pixel_values
														for idx, layers in enumerate(zip(self.patch_embeddings  , self.block  )  ):
																					UpperCamelCase__     ,     UpperCamelCase__				     =			layers
																					# Get patch embeddings from hidden_states
																					UpperCamelCase__				     =			embedding_layer(__lowerCAmelCase  )
																					# Send the embeddings through the blocks
																					for _, blk in enumerate(__lowerCAmelCase  ):
																												UpperCamelCase__				     =			blk(__lowerCAmelCase  )
																												UpperCamelCase__				     =			layer_outputs[0]

																					if output_hidden_states:
																												UpperCamelCase__				     =			all_hidden_states + (hidden_states,)

														if not return_dict:
																					return tuple(v for v in [hidden_states, all_hidden_states] if v is not None  )

														return BaseModelOutputWithNoAttention(last_hidden_state=__lowerCAmelCase  , hidden_states=__lowerCAmelCase  )






class        __SCREAMING_SNAKE_CASE					(     _a       ):
							snake_case     :      Union[str, Any]						     =       PoolFormerConfig
							snake_case     :      List[str]						     =       """poolformer"""
							snake_case     :      Union[str, Any]						     =       """pixel_values"""
							snake_case     :      Optional[Any]						     =       True






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if isinstance(__lowerCAmelCase  , (nn.Linear, nn.Convad)  ):
																					module.weight.data.normal_(mean=0.0  , std=self.config.initializer_range  )
																					if module.bias is not None:
																												module.bias.data.zero_()
														elif isinstance(__lowerCAmelCase  , nn.LayerNorm  ):
																					module.bias.data.zero_()
																					module.weight.data.fill_(1.0  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase=False  ):
														if isinstance(__lowerCAmelCase  , __lowerCAmelCase  ):
																					UpperCamelCase__				     =			value






UpperCamelCase__											= r"\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

UpperCamelCase__											= r"\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`PoolFormerImageProcessor.__call__`] for details.\n"

@add_start_docstrings(
    """The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top."""   ,		_a   ,		)
class        __SCREAMING_SNAKE_CASE					(     _a       ):






							def __init__(						self  , __lowerCAmelCase  ):
														super().__init__(__lowerCAmelCase  )
														UpperCamelCase__				     =			config

														UpperCamelCase__				     =			PoolFormerEncoder(__lowerCAmelCase  )

														# Initialize weights and apply final processing
														self.post_init()






							def 				_lowerCamelCase		(						self  ):
														return self.embeddings.patch_embeddings







							@add_start_docstrings_to_model_forward(__lowerCAmelCase  )
							@add_code_sample_docstrings(
							    checkpoint=_CHECKPOINT_FOR_DOC  , output_type=__lowerCAmelCase  , config_class=_CONFIG_FOR_DOC  , modality="""vision"""  , expected_output=_EXPECTED_OUTPUT_SHAPE  , )
							def 				_lowerCamelCase		(						self  , __lowerCAmelCase = None  , __lowerCAmelCase = None  , __lowerCAmelCase = None  , ):
														UpperCamelCase__				     =			(
														    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
														)
														UpperCamelCase__				     =			return_dict if return_dict is not None else self.config.use_return_dict

														if pixel_values is None:
																					raise ValueError("""You have to specify pixel_values"""  )

														UpperCamelCase__				     =			self.encoder(
														    __lowerCAmelCase  , output_hidden_states=__lowerCAmelCase  , return_dict=__lowerCAmelCase  , )
														UpperCamelCase__				     =			encoder_outputs[0]

														if not return_dict:
																					return (sequence_output, None) + encoder_outputs[1:]

														return BaseModelOutputWithNoAttention(
														    last_hidden_state=__lowerCAmelCase  , hidden_states=encoder_outputs.hidden_states  , )






class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  , __lowerCAmelCase  ):
														super().__init__()
														UpperCamelCase__				     =			nn.Linear(config.hidden_size  , config.hidden_size  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.dense(__lowerCAmelCase  )
														return output






@add_start_docstrings(
    """
    PoolFormer Model transformer with an image classification head on top
    """   ,		_a   ,		)
class        __SCREAMING_SNAKE_CASE					(     _a       ):






							def __init__(						self  , __lowerCAmelCase  ):
														super().__init__(__lowerCAmelCase  )
														UpperCamelCase__				     =			config.num_labels
														UpperCamelCase__				     =			PoolFormerModel(__lowerCAmelCase  )

														# Final norm
														UpperCamelCase__				     =			PoolFormerGroupNorm(config.hidden_sizes[-1]  )
														# Classifier head
														UpperCamelCase__				     =			(
														    nn.Linear(config.hidden_sizes[-1]  , config.num_labels  ) if config.num_labels > 0 else nn.Identity()
														)

														# Initialize weights and apply final processing
														self.post_init()







							@add_start_docstrings_to_model_forward(__lowerCAmelCase  )
							@add_code_sample_docstrings(
							    checkpoint=_IMAGE_CLASS_CHECKPOINT  , output_type=__lowerCAmelCase  , config_class=_CONFIG_FOR_DOC  , expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT  , )
							def 				_lowerCamelCase		(						self  , __lowerCAmelCase = None  , __lowerCAmelCase = None  , __lowerCAmelCase = None  , __lowerCAmelCase = None  , ):
														UpperCamelCase__				     =			return_dict if return_dict is not None else self.config.use_return_dict

														UpperCamelCase__				     =			self.poolformer(
														    __lowerCAmelCase  , output_hidden_states=__lowerCAmelCase  , return_dict=__lowerCAmelCase  , )

														UpperCamelCase__				     =			outputs[0]

														UpperCamelCase__				     =			self.classifier(self.norm(__lowerCAmelCase  ).mean([-2, -1]  )  )

														UpperCamelCase__				     =			None
														if labels is not None:
																					if self.config.problem_type is None:
																												if self.num_labels == 1:
																																			UpperCamelCase__				     =			"""regression"""
																												elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																																			UpperCamelCase__				     =			"""single_label_classification"""
																												else:
																																			UpperCamelCase__				     =			"""multi_label_classification"""

																					if self.config.problem_type == "regression":
																												UpperCamelCase__				     =			MSELoss()
																												if self.num_labels == 1:
																																			UpperCamelCase__				     =			loss_fct(logits.squeeze()  , labels.squeeze()  )
																												else:
																																			UpperCamelCase__				     =			loss_fct(__lowerCAmelCase  , __lowerCAmelCase  )
																					elif self.config.problem_type == "single_label_classification":
																												UpperCamelCase__				     =			CrossEntropyLoss()
																												UpperCamelCase__				     =			loss_fct(logits.view(-1  , self.num_labels  )  , labels.view(-1  )  )
																					elif self.config.problem_type == "multi_label_classification":
																												UpperCamelCase__				     =			BCEWithLogitsLoss()
																												UpperCamelCase__				     =			loss_fct(__lowerCAmelCase  , __lowerCAmelCase  )

														if not return_dict:
																					UpperCamelCase__				     =			(logits,) + outputs[2:]
																					return ((loss,) + output) if loss is not None else output

														return ImageClassifierOutputWithNoAttention(loss=__lowerCAmelCase  , logits=__lowerCAmelCase  , hidden_states=outputs.hidden_states  )











import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):
							snake_case     :      Any						     =       MODEL_FOR_MASKED_LM_MAPPING
							snake_case     :      Dict						     =       TF_MODEL_FOR_MASKED_LM_MAPPING






							def 				_lowerCamelCase		(						self  ):
														super().tearDown()
														# clean-up as much as possible GPU memory occupied by PyTorch
														gc.collect()
														if is_torch_available():
																					import torch

																					torch.cuda.empty_cache()






							@require_tf
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , top_k=2  , framework="""tf"""  )
														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is grouped""", """score""": 2.1E-05, """token""": 38015, """token_str""": """ grouped"""},
														        {"""sequence""": """My name is accuser""", """score""": 2.1E-05, """token""": 25506, """token_str""": """ accuser"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""The largest city in France is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {
														            """sequence""": """The largest city in France is grouped""",
														            """score""": 2.1E-05,
														            """token""": 38015,
														            """token_str""": """ grouped""",
														        },
														        {
														            """sequence""": """The largest city in France is accuser""",
														            """score""": 2.1E-05,
														            """token""": 25506,
														            """token_str""": """ accuser""",
														        },
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  , targets=[""" Patrick""", """ Clara""", """ Teven"""]  , top_k=3  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 13606, """token_str""": """ Clara"""},
														        {"""sequence""": """My name is Patrick""", """score""": 2E-05, """token""": 3499, """token_str""": """ Patrick"""},
														        {"""sequence""": """My name is Te""", """score""": 1.9E-05, """token""": 2941, """token_str""": """ Te"""},
														    ]  , )






							@require_torch
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , top_k=2  , framework="""pt"""  )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is Maul""", """score""": 2.2E-05, """token""": 35676, """token_str""": """ Maul"""},
														        {"""sequence""": """My name isELS""", """score""": 2.2E-05, """token""": 16416, """token_str""": """ELS"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""The largest city in France is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {
														            """sequence""": """The largest city in France is Maul""",
														            """score""": 2.2E-05,
														            """token""": 35676,
														            """token_str""": """ Maul""",
														        },
														        {"""sequence""": """The largest city in France isELS""", """score""": 2.2E-05, """token""": 16416, """token_str""": """ELS"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  , targets=[""" Patrick""", """ Clara""", """ Teven"""]  , top_k=3  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is Patrick""", """score""": 2.1E-05, """token""": 3499, """token_str""": """ Patrick"""},
														        {"""sequence""": """My name is Te""", """score""": 2E-05, """token""": 2941, """token_str""": """ Te"""},
														        {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 13606, """token_str""": """ Clara"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask> <mask>"""  , top_k=2  )

														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        [
														            {
														                """score""": 2.2E-05,
														                """token""": 35676,
														                """token_str""": """ Maul""",
														                """sequence""": """<s>My name is Maul<mask></s>""",
														            },
														            {"""score""": 2.2E-05, """token""": 16416, """token_str""": """ELS""", """sequence""": """<s>My name isELS<mask></s>"""},
														        ],
														        [
														            {
														                """score""": 2.2E-05,
														                """token""": 35676,
														                """token_str""": """ Maul""",
														                """sequence""": """<s>My name is<mask> Maul</s>""",
														            },
														            {"""score""": 2.2E-05, """token""": 16416, """token_str""": """ELS""", """sequence""": """<s>My name is<mask>ELS</s>"""},
														        ],
														    ]  , )






							@require_torch_gpu
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline("""fill-mask"""  , model="""hf-internal-testing/tiny-random-distilbert"""  , device=0  , framework="""pt"""  )

														# convert model to fp16
														pipe.model.half()

														UpperCamelCase__				     =			pipe("""Paris is the [MASK] of France."""  )
														# We actually don't care about the result, we just want to make sure
														# it works, meaning the float16 tensor got casted back to float32
														# for postprocessing.
														self.assertIsInstance(__lowerCAmelCase  , __lowerCAmelCase  )






							@slow
							@require_torch
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""distilroberta-base"""  , top_k=2  , framework="""pt"""  )
														self.run_large_test(__lowerCAmelCase  )






							@slow
							@require_tf
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""distilroberta-base"""  , top_k=2  , framework="""tf"""  )
														self.run_large_test(__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  )  , [
														        {"""sequence""": """My name is John""", """score""": 0.008, """token""": 610, """token_str""": """ John"""},
														        {"""sequence""": """My name is Chris""", """score""": 0.007, """token""": 1573, """token_str""": """ Chris"""},
														    ]  , )
														UpperCamelCase__				     =			unmasker("""The largest city in France is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  )  , [
														        {
														            """sequence""": """The largest city in France is Paris""",
														            """score""": 0.251,
														            """token""": 2201,
														            """token_str""": """ Paris""",
														        },
														        {
														            """sequence""": """The largest city in France is Lyon""",
														            """score""": 0.214,
														            """token""": 12790,
														            """token_str""": """ Lyon""",
														        },
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  , targets=[""" Patrick""", """ Clara""", """ Teven"""]  , top_k=3  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  )  , [
														        {"""sequence""": """My name is Patrick""", """score""": 0.005, """token""": 3499, """token_str""": """ Patrick"""},
														        {"""sequence""": """My name is Clara""", """score""": 0.000, """token""": 13606, """token_str""": """ Clara"""},
														        {"""sequence""": """My name is Te""", """score""": 0.000, """token""": 2941, """token_str""": """ Te"""},
														    ]  , )






							@require_torch
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , framework="""pt"""  )
														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			None
														self.run_pipeline_test(__lowerCAmelCase  , []  )






							@require_tf
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , framework="""tf"""  )
														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			None
														self.run_pipeline_test(__lowerCAmelCase  , []  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														if tokenizer is None or tokenizer.mask_token_id is None:
																					self.skipTest("""The provided tokenizer has no mask token, (probably reformer or wav2vec2)"""  )

														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			[
														    f"""This is another {tokenizer.mask_token} test""",
														]
														return fill_masker, examples






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			fill_masker.tokenizer
														UpperCamelCase__				     =			fill_masker.model

														UpperCamelCase__				     =			fill_masker(
														    f"""This is a {tokenizer.mask_token}"""  , )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )

														UpperCamelCase__				     =			fill_masker([f"""This is a {tokenizer.mask_token}"""]  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )

														UpperCamelCase__				     =			fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														    ]  , )

														with self.assertRaises(__lowerCAmelCase  ):
																					fill_masker([None]  )
														# No mask_token is not supported
														with self.assertRaises(__lowerCAmelCase  ):
																					fill_masker("""This is"""  )

														self.run_test_top_k(__lowerCAmelCase  , __lowerCAmelCase  )
														self.run_test_targets(__lowerCAmelCase  , __lowerCAmelCase  )
														self.run_test_top_k_targets(__lowerCAmelCase  , __lowerCAmelCase  )
														self.fill_mask_with_duplicate_targets_and_top_k(__lowerCAmelCase  , __lowerCAmelCase  )
														self.fill_mask_with_multiple_masks(__lowerCAmelCase  , __lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			tokenizer.get_vocab()
														UpperCamelCase__				     =			sorted(vocab.keys()  )[:2]
														# Pipeline argument
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  , targets=__lowerCAmelCase  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )
														UpperCamelCase__				     =			{vocab[el] for el in targets}
														self.assertEqual({el["""token"""] for el in outputs}  , __lowerCAmelCase  )
														UpperCamelCase__				     =			[tokenizer.decode([x]  ) for x in target_ids]
														self.assertEqual({el["""token_str"""] for el in outputs}  , set(__lowerCAmelCase  )  )

														# Call argument
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )
														UpperCamelCase__				     =			{vocab[el] for el in targets}
														self.assertEqual({el["""token"""] for el in outputs}  , __lowerCAmelCase  )
														UpperCamelCase__				     =			[tokenizer.decode([x]  ) for x in target_ids]
														self.assertEqual({el["""token_str"""] for el in outputs}  , set(__lowerCAmelCase  )  )

														# Score equivalence
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  )
														UpperCamelCase__				     =			[top_mask["""token_str"""] for top_mask in outputs]
														UpperCamelCase__				     =			[top_mask["""score"""] for top_mask in outputs]

														# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
														if set(__lowerCAmelCase  ) == set(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  )
																					UpperCamelCase__				     =			[top_mask["""score"""] for top_mask in unmasked_targets]
																					self.assertEqual(nested_simplify(__lowerCAmelCase  )  , nested_simplify(__lowerCAmelCase  )  )

														# Raises with invalid
														with self.assertRaises(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=[]  )
														# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
														if "" not in tokenizer.get_vocab():
																					with self.assertRaises(__lowerCAmelCase  ):
																												UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=[""""""]  )
																					with self.assertRaises(__lowerCAmelCase  ):
																												UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=""""""  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  , top_k=2  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )

														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , top_k=2  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )
														self.assertEqual(nested_simplify(__lowerCAmelCase  )  , nested_simplify(__lowerCAmelCase  )  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			tokenizer.get_vocab()
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )

														# top_k=2, ntargets=3
														UpperCamelCase__				     =			sorted(vocab.keys()  )[:3]
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , top_k=2  , targets=__lowerCAmelCase  )

														# If we use the most probably targets, and filter differently, we should still
														# have the same results
														UpperCamelCase__				     =			[el["""token_str"""] for el in sorted(__lowerCAmelCase  , key=lambda __lowerCAmelCase  : x["score"]  , reverse=__lowerCAmelCase  )]
														# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
														if set(__lowerCAmelCase  ).issubset(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , top_k=3  , targets=__lowerCAmelCase  )
																					# They should yield exactly the same result
																					self.assertEqual(nested_simplify(__lowerCAmelCase  )  , nested_simplify(__lowerCAmelCase  )  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			tokenizer.get_vocab()
														# String duplicates + id duplicates
														UpperCamelCase__				     =			sorted(vocab.keys()  )[:3]
														UpperCamelCase__				     =			[targets[0], targets[1], targets[0], targets[2], targets[1]]
														UpperCamelCase__				     =			fill_masker(f"""My name is {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  , top_k=10  )

														# The target list contains duplicates, so we can't output more
														# than them
														self.assertEqual(len(__lowerCAmelCase  )  , 3  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )

														UpperCamelCase__				     =			fill_masker(
														    f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""  , top_k=2  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														    ]  , )




