





'''simple docstring'''






from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


A__							:  Any    =logging.get_logger(__name__)

# General docstring
A__							:  Optional[Any]    ='''RegNetConfig'''

# Base docstring
A__							:  Dict    ='''facebook/regnet-y-040'''
A__							:  str    =[1, 10_88, 7, 7]

# Image classification docstring
A__							:  Union[str, Any]    ='''facebook/regnet-y-040'''
A__							:  int    ='''tabby, tabby cat'''

A__							:  Optional[Any]    =[
    '''facebook/regnet-y-040''',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					str     ,     __snake_case     :					int     ,     __snake_case     :					int     ,     __snake_case     :					int = 3     ,     __snake_case     :					int = 1     ,     __snake_case     :					int = 1     ,     __snake_case     :					Optional[str] = "relu"     ,     )							-> Optional[Any]:
            super().__init__()
            _lowerCAmelCase					=				nn.Convad(
                __snake_case     ,     __snake_case     ,     kernel_size=__snake_case     ,     stride=__snake_case     ,     padding=kernel_size // 2     ,     groups=__snake_case     ,     bias=__snake_case     ,     )
            _lowerCAmelCase					=				nn.BatchNormad(__snake_case    )
            _lowerCAmelCase					=				ACTaFN[activation] if activation is not None else nn.Identity()




      def 							lowercase__       (     self     :					List[Any]     ,     __snake_case     :					Dict    )							-> List[str]:
            _lowerCAmelCase					=				self.convolution(__snake_case    )
            _lowerCAmelCase					=				self.normalization(__snake_case    )
            _lowerCAmelCase					=				self.activation(__snake_case    )
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					Dict     ,     __snake_case     :					RegNetConfig    )							-> Union[str, Any]:
            super().__init__()
            _lowerCAmelCase					=				RegNetConvLayer(
                config.num_channels     ,     config.embedding_size     ,     kernel_size=3     ,     stride=2     ,     activation=config.hidden_act    )
            _lowerCAmelCase					=				config.num_channels




      def 							lowercase__       (     self     :					Union[str, Any]     ,     __snake_case     :					Union[str, Any]    )							-> Optional[Any]:
            _lowerCAmelCase					=				pixel_values.shape[1]
            if num_channels != self.num_channels:
                  raise ValueError(
                      """Make sure that the channel dimension of the pixel values match with the one set in the configuration."""    )
            _lowerCAmelCase					=				self.embedder(__snake_case    )
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					Union[str, Any]     ,     __snake_case     :					int     ,     __snake_case     :					int     ,     __snake_case     :					int = 2    )							-> Optional[int]:
            super().__init__()
            _lowerCAmelCase					=				nn.Convad(__snake_case     ,     __snake_case     ,     kernel_size=1     ,     stride=__snake_case     ,     bias=__snake_case    )
            _lowerCAmelCase					=				nn.BatchNormad(__snake_case    )




      def 							lowercase__       (     self     :					int     ,     __snake_case     :					Tensor    )							-> Tensor:
            _lowerCAmelCase					=				self.convolution(__snake_case    )
            _lowerCAmelCase					=				self.normalization(__snake_case    )
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					Optional[int]     ,     __snake_case     :					int     ,     __snake_case     :					int    )							-> int:
            super().__init__()

            _lowerCAmelCase					=				nn.AdaptiveAvgPoolad((1, 1)    )
            _lowerCAmelCase					=				nn.Sequential(
                nn.Convad(__snake_case     ,     __snake_case     ,     kernel_size=1    )     ,     nn.ReLU()     ,     nn.Convad(__snake_case     ,     __snake_case     ,     kernel_size=1    )     ,     nn.Sigmoid()     ,     )




      def 							lowercase__       (     self     :					List[str]     ,     __snake_case     :					List[str]    )							-> Optional[Any]:
            # b c h w -> b c 1 1
            _lowerCAmelCase					=				self.pooler(__snake_case    )
            _lowerCAmelCase					=				self.attention(__snake_case    )
            _lowerCAmelCase					=				hidden_state * attention
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					Tuple     ,     __snake_case     :					RegNetConfig     ,     __snake_case     :					int     ,     __snake_case     :					int     ,     __snake_case     :					int = 1    )							-> Optional[int]:
            super().__init__()
            _lowerCAmelCase					=				in_channels != out_channels or stride != 1
            _lowerCAmelCase					=				max(1     ,     out_channels // config.groups_width    )
            _lowerCAmelCase					=				(
                RegNetShortCut(__snake_case     ,     __snake_case     ,     stride=__snake_case    ) if should_apply_shortcut else nn.Identity()
            )
            _lowerCAmelCase					=				nn.Sequential(
                RegNetConvLayer(__snake_case     ,     __snake_case     ,     kernel_size=1     ,     activation=config.hidden_act    )     ,     RegNetConvLayer(__snake_case     ,     __snake_case     ,     stride=__snake_case     ,     groups=__snake_case     ,     activation=config.hidden_act    )     ,     RegNetConvLayer(__snake_case     ,     __snake_case     ,     kernel_size=1     ,     activation=__snake_case    )     ,     )
            _lowerCAmelCase					=				ACTaFN[config.hidden_act]




      def 							lowercase__       (     self     :					int     ,     __snake_case     :					str    )							-> List[Any]:
            _lowerCAmelCase					=				hidden_state
            _lowerCAmelCase					=				self.layer(__snake_case    )
            _lowerCAmelCase					=				self.shortcut(__snake_case    )
            hidden_state += residual
            _lowerCAmelCase					=				self.activation(__snake_case    )
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					str     ,     __snake_case     :					RegNetConfig     ,     __snake_case     :					int     ,     __snake_case     :					int     ,     __snake_case     :					int = 1    )							-> Optional[Any]:
            super().__init__()
            _lowerCAmelCase					=				in_channels != out_channels or stride != 1
            _lowerCAmelCase					=				max(1     ,     out_channels // config.groups_width    )
            _lowerCAmelCase					=				(
                RegNetShortCut(__snake_case     ,     __snake_case     ,     stride=__snake_case    ) if should_apply_shortcut else nn.Identity()
            )
            _lowerCAmelCase					=				nn.Sequential(
                RegNetConvLayer(__snake_case     ,     __snake_case     ,     kernel_size=1     ,     activation=config.hidden_act    )     ,     RegNetConvLayer(__snake_case     ,     __snake_case     ,     stride=__snake_case     ,     groups=__snake_case     ,     activation=config.hidden_act    )     ,     RegNetSELayer(__snake_case     ,     reduced_channels=int(round(in_channels / 4    )    )    )     ,     RegNetConvLayer(__snake_case     ,     __snake_case     ,     kernel_size=1     ,     activation=__snake_case    )     ,     )
            _lowerCAmelCase					=				ACTaFN[config.hidden_act]




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					Optional[Any]    )							-> Union[str, Any]:
            _lowerCAmelCase					=				hidden_state
            _lowerCAmelCase					=				self.layer(__snake_case    )
            _lowerCAmelCase					=				self.shortcut(__snake_case    )
            hidden_state += residual
            _lowerCAmelCase					=				self.activation(__snake_case    )
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					List[Any]     ,     __snake_case     :					RegNetConfig     ,     __snake_case     :					int     ,     __snake_case     :					int     ,     __snake_case     :					int = 2     ,     __snake_case     :					int = 2     ,     )							-> str:
            super().__init__()

            _lowerCAmelCase					=				RegNetXLayer if config.layer_type == """x""" else RegNetYLayer

            _lowerCAmelCase					=				nn.Sequential(
                # downsampling is done in the first layer with stride of 2
                layer(
                    __snake_case     ,     __snake_case     ,     __snake_case     ,     stride=__snake_case     ,     )     ,     *[layer(__snake_case     ,     __snake_case     ,     __snake_case    ) for _ in range(depth - 1    )]     ,     )




      def 							lowercase__       (     self     :					str     ,     __snake_case     :					Tuple    )							-> List[str]:
            _lowerCAmelCase					=				self.layers(__snake_case    )
            return hidden_state




class 					UpperCAmelCase	(     nn.Module  ):




      def __init__(     self     :					Optional[int]     ,     __snake_case     :					RegNetConfig    )							-> int:
            super().__init__()
            _lowerCAmelCase					=				nn.ModuleList([]    )
            # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
            self.stages.append(
                RegNetStage(
                    __snake_case     ,     config.embedding_size     ,     config.hidden_sizes[0]     ,     stride=2 if config.downsample_in_first_stage else 1     ,     depth=config.depths[0]     ,     )    )
            _lowerCAmelCase					=				zip(config.hidden_sizes     ,     config.hidden_sizes[1:]    )
            for (in_channels, out_channels), depth in zip(__snake_case     ,     config.depths[1:]    ):
                  self.stages.append(RegNetStage(__snake_case     ,     __snake_case     ,     __snake_case     ,     depth=__snake_case    )    )




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					Tensor     ,     __snake_case     :					bool = False     ,     __snake_case     :					bool = True    )							-> BaseModelOutputWithNoAttention:
            _lowerCAmelCase					=				() if output_hidden_states else None

            for stage_module in self.stages:
                  if output_hidden_states:
                        _lowerCAmelCase					=				hidden_states + (hidden_state,)

                  _lowerCAmelCase					=				stage_module(__snake_case    )

            if output_hidden_states:
                  _lowerCAmelCase					=				hidden_states + (hidden_state,)

            if not return_dict:
                  return tuple(v for v in [hidden_state, hidden_states] if v is not None    )

            return BaseModelOutputWithNoAttention(last_hidden_state=__snake_case     ,     hidden_states=__snake_case    )




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     List[Any]			=  RegNetConfig
      _lowercase:     Optional[int]			=  '''regnet'''
      _lowercase:     List[str]			=  '''pixel_values'''
      _lowercase:     Tuple			=  True




      def 							lowercase__       (     self     :					Union[str, Any]     ,     __snake_case     :					Optional[Any]    )							-> Tuple:
            if isinstance(__snake_case     ,     nn.Convad    ):
                  nn.init.kaiming_normal_(module.weight     ,     mode="""fan_out"""     ,     nonlinearity="""relu"""    )
            elif isinstance(__snake_case     ,     (nn.BatchNormad, nn.GroupNorm)    ):
                  nn.init.constant_(module.weight     ,     1    )
                  nn.init.constant_(module.bias     ,     0    )




      def 							lowercase__       (     self     :					Tuple     ,     __snake_case     :					List[Any]     ,     __snake_case     :					Any=False    )							-> str:
            if isinstance(__snake_case     ,     __snake_case    ):
                  _lowerCAmelCase					=				value




A__							:  Dict    =r'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

A__							:  int    =r'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
'''




@add_start_docstrings(
    '''The bare RegNet model outputting raw features without any specific head on top.'''				,						snake_case_				,						)
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class 					UpperCAmelCase	(     snake_case_  ):




      def __init__(     self     :					int     ,     __snake_case     :					Optional[int]    )							-> Union[str, Any]:
            super().__init__(__snake_case    )
            _lowerCAmelCase					=				config
            _lowerCAmelCase					=				RegNetEmbeddings(__snake_case    )
            _lowerCAmelCase					=				RegNetEncoder(__snake_case    )
            _lowerCAmelCase					=				nn.AdaptiveAvgPoolad((1, 1)    )
            # Initialize weights and apply final processing
            self.post_init()




      @add_start_docstrings_to_model_forward(__snake_case    )
      @add_code_sample_docstrings(
          checkpoint=_CHECKPOINT_FOR_DOC     ,     output_type=__snake_case     ,     config_class=_CONFIG_FOR_DOC     ,     modality="""vision"""     ,     expected_output=_EXPECTED_OUTPUT_SHAPE     ,     )
      def 							lowercase__       (     self     :					Union[str, Any]     ,     __snake_case     :					Tensor     ,     __snake_case     :					Optional[bool] = None     ,     __snake_case     :					Optional[bool] = None    )							-> BaseModelOutputWithPoolingAndNoAttention:
            _lowerCAmelCase					=				(
                output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
            )
            _lowerCAmelCase					=				return_dict if return_dict is not None else self.config.use_return_dict

            _lowerCAmelCase					=				self.embedder(__snake_case    )

            _lowerCAmelCase					=				self.encoder(
                __snake_case     ,     output_hidden_states=__snake_case     ,     return_dict=__snake_case    )

            _lowerCAmelCase					=				encoder_outputs[0]

            _lowerCAmelCase					=				self.pooler(__snake_case    )

            if not return_dict:
                  return (last_hidden_state, pooled_output) + encoder_outputs[1:]

            return BaseModelOutputWithPoolingAndNoAttention(
                last_hidden_state=__snake_case     ,     pooler_output=__snake_case     ,     hidden_states=encoder_outputs.hidden_states     ,     )




@add_start_docstrings(
    '''
    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    '''				,						snake_case_				,						)
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class 					UpperCAmelCase	(     snake_case_  ):




      def __init__(     self     :					List[Any]     ,     __snake_case     :					int    )							-> str:
            super().__init__(__snake_case    )
            _lowerCAmelCase					=				config.num_labels
            _lowerCAmelCase					=				RegNetModel(__snake_case    )
            # classification head
            _lowerCAmelCase					=				nn.Sequential(
                nn.Flatten()     ,     nn.Linear(config.hidden_sizes[-1]     ,     config.num_labels    ) if config.num_labels > 0 else nn.Identity()     ,     )
            # initialize weights and apply final processing
            self.post_init()




      @add_start_docstrings_to_model_forward(__snake_case    )
      @add_code_sample_docstrings(
          checkpoint=_IMAGE_CLASS_CHECKPOINT     ,     output_type=__snake_case     ,     config_class=_CONFIG_FOR_DOC     ,     expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT     ,     )
      def 							lowercase__       (     self     :					List[str]     ,     __snake_case     :					Optional[torch.FloatTensor] = None     ,     __snake_case     :					Optional[torch.LongTensor] = None     ,     __snake_case     :					Optional[bool] = None     ,     __snake_case     :					Optional[bool] = None     ,     )							-> ImageClassifierOutputWithNoAttention:
            _lowerCAmelCase					=				return_dict if return_dict is not None else self.config.use_return_dict

            _lowerCAmelCase					=				self.regnet(__snake_case     ,     output_hidden_states=__snake_case     ,     return_dict=__snake_case    )

            _lowerCAmelCase					=				outputs.pooler_output if return_dict else outputs[1]

            _lowerCAmelCase					=				self.classifier(__snake_case    )

            _lowerCAmelCase					=				None

            if labels is not None:
                  if self.config.problem_type is None:
                        if self.num_labels == 1:
                              _lowerCAmelCase					=				"""regression"""
                        elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                              _lowerCAmelCase					=				"""single_label_classification"""
                        else:
                              _lowerCAmelCase					=				"""multi_label_classification"""
                  if self.config.problem_type == "regression":
                        _lowerCAmelCase					=				MSELoss()
                        if self.num_labels == 1:
                              _lowerCAmelCase					=				loss_fct(logits.squeeze()     ,     labels.squeeze()    )
                        else:
                              _lowerCAmelCase					=				loss_fct(__snake_case     ,     __snake_case    )
                  elif self.config.problem_type == "single_label_classification":
                        _lowerCAmelCase					=				CrossEntropyLoss()
                        _lowerCAmelCase					=				loss_fct(logits.view(-1     ,     self.num_labels    )     ,     labels.view(-1    )    )
                  elif self.config.problem_type == "multi_label_classification":
                        _lowerCAmelCase					=				BCEWithLogitsLoss()
                        _lowerCAmelCase					=				loss_fct(__snake_case     ,     __snake_case    )

            if not return_dict:
                  _lowerCAmelCase					=				(logits,) + outputs[2:]
                  return (loss,) + output if loss is not None else output

            return ImageClassifierOutputWithNoAttention(loss=__snake_case     ,     logits=__snake_case     ,     hidden_states=outputs.hidden_states    )













'''simple docstring'''






import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin




class 					UpperCAmelCase	(     snake_case_				,						unittest.TestCase  ):
      _lowercase:     Optional[int]			=  ProphetNetTokenizer
      _lowercase:     Union[str, Any]			=  False




      def 							lowercase__       (     self     :					Dict    )							-> Optional[int]:
            super().setUp()

            _lowerCAmelCase					=				[
                """[UNK]""",
                """[CLS]""",
                """[SEP]""",
                """[PAD]""",
                """[MASK]""",
                """want""",
                """##want""",
                """##ed""",
                """wa""",
                """un""",
                """runn""",
                """##ing""",
                """,""",
                """low""",
                """lowest""",
            ]
            _lowerCAmelCase					=				os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""vocab_file"""]    )
            with open(self.vocab_file     ,     """w"""     ,     encoding="""utf-8"""    ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )




      def 							lowercase__       (     self     :					Tuple     ,     __snake_case     :					str    )							-> List[str]:
            _lowerCAmelCase					=				"""UNwant\u00E9d,running"""
            _lowerCAmelCase					=				"""unwanted, running"""
            return input_text, output_text




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> Optional[int]:
            _lowerCAmelCase					=				self.tokenizer_class(self.vocab_file    )

            _lowerCAmelCase					=				tokenizer.tokenize("""UNwant\u00E9d,running"""    )
            self.assertListEqual(__snake_case     ,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case    )     ,     [9, 6, 7, 12, 10, 11]    )




      def 							lowercase__       (     self     :					List[Any]    )							-> List[str]:
            _lowerCAmelCase					=				BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )     ,     ["""ah""", """\u535A""", """\u63A8""", """zz"""]    )




      def 							lowercase__       (     self     :					Optional[Any]    )							-> List[Any]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )     ,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""hello"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> int:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""h\u00E9llo"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> int:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""hello"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> Optional[Any]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )     ,     ["""hello"""]    )




      def 							lowercase__       (     self     :					str    )							-> List[str]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )     ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> str:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )




      def 							lowercase__       (     self     :					Tuple    )							-> str:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     strip_accents=__snake_case    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )     ,     ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )




      def 							lowercase__       (     self     :					Dict    )							-> Optional[int]:
            _lowerCAmelCase					=				BasicTokenizer(do_lower_case=__snake_case     ,     never_split=["""[UNK]"""]    )

            self.assertListEqual(
                tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )     ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )




      def 							lowercase__       (     self     :					Any    )							-> Dict:
            _lowerCAmelCase					=				["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

            _lowerCAmelCase					=				{}
            for i, token in enumerate(__snake_case    ):
                  _lowerCAmelCase					=				i
            _lowerCAmelCase					=				WordpieceTokenizer(vocab=__snake_case     ,     unk_token="""[UNK]"""    )

            self.assertListEqual(tokenizer.tokenize(""""""    )     ,     []    )

            self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )     ,     ["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

            self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )     ,     ["""[UNK]""", """runn""", """##ing"""]    )




      @require_torch
      def 							lowercase__       (     self     :					Optional[int]    )							-> Optional[Any]:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""    )

            _lowerCAmelCase					=				["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
            _lowerCAmelCase					=				[10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
            _lowerCAmelCase					=				tokenizer(__snake_case     ,     padding=__snake_case     ,     return_tensors="""pt"""    )
            self.assertIsInstance(__snake_case     ,     __snake_case    )
            _lowerCAmelCase					=				list(batch.input_ids.numpy()[0]    )
            self.assertListEqual(__snake_case     ,     __snake_case    )

            self.assertEqual((2, 9)     ,     batch.input_ids.shape    )
            self.assertEqual((2, 9)     ,     batch.attention_mask.shape    )




      def 							lowercase__       (     self     :					List[Any]    )							-> Any:
            self.assertTrue(_is_whitespace(""" """    )    )
            self.assertTrue(_is_whitespace("""\t"""    )    )
            self.assertTrue(_is_whitespace("""\r"""    )    )
            self.assertTrue(_is_whitespace("""\n"""    )    )
            self.assertTrue(_is_whitespace("""\u00A0"""    )    )

            self.assertFalse(_is_whitespace("""A"""    )    )
            self.assertFalse(_is_whitespace("""-"""    )    )




      def 							lowercase__       (     self     :					str    )							-> Union[str, Any]:
            self.assertTrue(_is_control("""\u0005"""    )    )

            self.assertFalse(_is_control("""A"""    )    )
            self.assertFalse(_is_control(""" """    )    )
            self.assertFalse(_is_control("""\t"""    )    )
            self.assertFalse(_is_control("""\r"""    )    )




      def 							lowercase__       (     self     :					int    )							-> int:
            self.assertTrue(_is_punctuation("""-"""    )    )
            self.assertTrue(_is_punctuation("""$"""    )    )
            self.assertTrue(_is_punctuation("""`"""    )    )
            self.assertTrue(_is_punctuation("""."""    )    )

            self.assertFalse(_is_punctuation("""A"""    )    )
            self.assertFalse(_is_punctuation(""" """    )    )




      @slow
      def 							lowercase__       (     self     :					int    )							-> Optional[int]:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""    )

            _lowerCAmelCase					=				tokenizer.encode("""sequence builders"""     ,     add_special_tokens=__snake_case    )
            _lowerCAmelCase					=				tokenizer.encode("""multi-sequence build"""     ,     add_special_tokens=__snake_case    )

            _lowerCAmelCase					=				tokenizer.build_inputs_with_special_tokens(__snake_case    )
            _lowerCAmelCase					=				tokenizer.build_inputs_with_special_tokens(__snake_case     ,     __snake_case    )

            assert encoded_sentence == text + [1_02]
            assert encoded_pair == text + [1_02] + text_a + [1_02]






