



"""simple docstring"""







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__		:    int				   =     logging.get_logger(__name__)

lowercase__		:    Union[str, Any]				   =     """▁"""

lowercase__		:    str				   =     {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

lowercase__		:    str				   =     {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

lowercase__		:    int				   =     {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

lowercase__		:    Union[str, Any]				   =     {
    """ernie-m-base""": 5_1_4,
    """ernie-m-large""": 5_1_4,
}

lowercase__		:    Optional[int]				   =     {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}







class 				UpperCamelCase__       (     lowercase_  ):




						"""simple docstring"""




						_SCREAMING_SNAKE_CASE      =					["""input_ids"""]

						_SCREAMING_SNAKE_CASE      =					VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE      =					PRETRAINED_INIT_CONFIGURATION
						_SCREAMING_SNAKE_CASE      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE      =					PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE      =					RESOURCE_FILES_NAMES



						def __init__(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				int   ,      SCREAMING_SNAKE_CASE_    :				Tuple=None   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]=False   ,      SCREAMING_SNAKE_CASE_    :				List[str]="utf8"   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]="[UNK]"   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]="[SEP]"   ,      SCREAMING_SNAKE_CASE_    :				int="[PAD]"   ,      SCREAMING_SNAKE_CASE_    :				List[Any]="[CLS]"   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]="[MASK]"   ,      SCREAMING_SNAKE_CASE_    :				Optional[Dict[str, Any]] = None   ,      **SCREAMING_SNAKE_CASE_    :				List[Any]   ,      ):
								lowerCAmelCase_       :    Any								=			{} if sp_model_kwargs is None else sp_model_kwargs
								super().__init__(
								    do_lower_case=lowercase_   ,      unk_token=lowercase_   ,      sep_token=lowercase_   ,      pad_token=lowercase_   ,      cls_token=lowercase_   ,      mask_token=lowercase_   ,      vocab_file=lowercase_   ,      encoding=lowercase_   ,      sp_model_kwargs=self.sp_model_kwargs   ,      **lowercase_   ,      )
								lowerCAmelCase_       :    Any								=			do_lower_case
								lowerCAmelCase_       :    Union[str, Any]								=			sentencepiece_model_ckpt
								lowerCAmelCase_       :    Tuple								=			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
								self.sp_model.Load(lowercase_	)

								# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
								if vocab_file is not None:
										lowerCAmelCase_       :    Union[str, Any]								=			self.load_vocab(filepath=lowercase_	)
								else:
										lowerCAmelCase_       :    Any								=			{self.sp_model.id_to_piece(lowercase_	): id for id in range(self.sp_model.get_piece_size()	)}
								lowerCAmelCase_       :    int								=			{v: k for k, v in self.vocab.items()}



						def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				Dict	):







								if text is None:
										return None

								lowerCAmelCase_       :    str								=			self.tokenize(lowercase_	)
								lowerCAmelCase_						,lowerCAmelCase_       :    Dict								=			'', []

								for i, ch in enumerate(lowercase_	):
										if ch in self.SP_CHAR_MAPPING:
												lowerCAmelCase_       :    Union[str, Any]								=			self.SP_CHAR_MAPPING.get(lowercase_	)
										else:
												lowerCAmelCase_       :    int								=			unicodedata.normalize('NFKC'   ,      lowercase_	)
										if self.is_whitespace(lowercase_	):
												continue
										normalized_text += ch
										char_mapping.extend([i] * len(lowercase_	)	)

								lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_       :    Union[str, Any]								=			normalized_text, [], 0

								if self.do_lower_case:
										lowerCAmelCase_       :    Optional[int]								=			text.lower()

								for token in split_tokens:
										if token[:1] == "▁":
												lowerCAmelCase_       :    List[Any]								=			token[1:]
										lowerCAmelCase_       :    Tuple								=			text[offset:].index(lowercase_	) + offset
										lowerCAmelCase_       :    Dict								=			start + len(lowercase_	)

										token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)	)
										lowerCAmelCase_       :    int								=			end
								return token_mapping



						@property
						def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
								return len(self.vocab	)



						def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
								return dict(self.vocab   ,      **self.added_tokens_encoder	)



						def __getstate__(				self    :				Dict	):
								lowerCAmelCase_       :    Tuple								=			self.__dict__.copy()
								lowerCAmelCase_       :    Union[str, Any]								=			None
								return state



						def __setstate__(				self    :				int   ,      SCREAMING_SNAKE_CASE_    :				Tuple	):
								lowerCAmelCase_       :    List[Any]								=			d

								# for backward compatibility
								if not hasattr(self   ,      'sp_model_kwargs'	):
										lowerCAmelCase_       :    Any								=			{}

								lowerCAmelCase_       :    Optional[Any]								=			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
								self.sp_model.Load(self.sentencepiece_model_ckpt	)



						def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Dict	):
								return "".join((self.SP_CHAR_MAPPING.get(lowercase_   ,      lowercase_	) for c in text)	)



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				Tuple=False   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=6_4   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=0.1	):







								if self.sp_model_kwargs.get('enable_sampling'	) is True:
										lowerCAmelCase_       :    Union[str, Any]								=			True
								if self.sp_model_kwargs.get('alpha'	) is not None:
										lowerCAmelCase_       :    List[str]								=			self.sp_model_kwargs.get('alpha'	)
								if self.sp_model_kwargs.get('nbest_size'	) is not None:
										lowerCAmelCase_       :    Optional[int]								=			self.sp_model_kwargs.get('nbest_size'	)

								if not enable_sampling:
										lowerCAmelCase_       :    Dict								=			self.sp_model.EncodeAsPieces(lowercase_	)
								else:
										lowerCAmelCase_       :    Optional[Any]								=			self.sp_model.SampleEncodeAsPieces(lowercase_   ,      lowercase_   ,      lowercase_	)
								lowerCAmelCase_       :    Optional[Any]								=			[]
								for pi, piece in enumerate(lowercase_	):
										if piece == SPIECE_UNDERLINE:
												if not pieces[pi + 1].startswith(lowercase_	) and pi != 0:
														new_pieces.append(lowercase_	)
														continue
												else:
														continue
										lowerCAmelCase_       :    str								=			0
										for i, chunk in enumerate(lowercase_	):
												if chunk == SPIECE_UNDERLINE:
														continue
												if self.is_ch_char(lowercase_	) or self.is_punct(lowercase_	):
														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]	)
														new_pieces.append(lowercase_	)
														lowerCAmelCase_       :    Any								=			i + 1
												elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]	)
														lowerCAmelCase_       :    Optional[Any]								=			i
												elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]	)
														lowerCAmelCase_       :    List[str]								=			i
										if len(lowercase_	) > lst_i:
												new_pieces.append(piece[lst_i:]	)
								return new_pieces



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]	):
								lowerCAmelCase_       :    Dict								=			''.join(lowercase_	).replace(lowercase_   ,      ' '	).strip()
								return out_string



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Dict	):
								lowerCAmelCase_       :    int								=			self.convert_ids_to_tokens(lowercase_	)
								lowerCAmelCase_       :    Any								=			''.join(lowercase_	).replace(lowercase_   ,      ' '	).strip()
								return out_string



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Any   ,      SCREAMING_SNAKE_CASE_    :				List[Any]	):
								return self.vocab.get(lowercase_   ,      self.vocab.get(self.unk_token	)	)



						def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]	):
								return self.reverse_vocab.get(lowercase_   ,      self.unk_token	)



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Tuple   ,      SCREAMING_SNAKE_CASE_    :				Tuple=None	):







								if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowerCAmelCase_       :    int								=			[self.cls_token_id]
								lowerCAmelCase_       :    Optional[int]								=			[self.sep_token_id]
								return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



						def 					SCREAMING_SNAKE_CASE__			(				self    :				int   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=None	):







								if offset_mapping_a is None:
										return [(0, 0)] + offset_mapping_a + [(0, 0)]

								return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



						def 					SCREAMING_SNAKE_CASE__			(				self    :				int   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=None   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=False	):







								if already_has_special_tokens:
										if token_ids_a is not None:
												raise ValueError(
												    'You should not supply a second sequence if the provided sequence of '
												    'ids is already formatted with special tokens for the model.'	)
										return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
										return [1] + ([0] * len(lowercase_	)) + [1, 1] + ([0] * len(lowercase_	)) + [1]
								return [1] + ([0] * len(lowercase_	)) + [1]



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				List[int]   ,      SCREAMING_SNAKE_CASE_    :				Optional[List[int]] = None	):







								if token_ids_a is None:
										# [CLS] X [SEP]
										return (len(lowercase_	) + 2) * [0]

								# [CLS] A [SEP] [SEP] B [SEP]
								return [0] * (len(lowercase_	) + 1) + [1] * (len(lowercase_	) + 3)



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				str	):







								if "\u4e00" <= char <= "\u9fff":
										return True
								return False



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]	):







								if ("a" <= char <= "z") or ("A" <= char <= "Z"):
										return True
								return False



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]	):







								if char in ",;:.?!~，；：。？！《》【】":
										return True
								return False



						def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				str	):







								if char == " " or char == "\t" or char == "\n" or char == "\r":
										return True
								if len(lowercase_	) == 1:
										lowerCAmelCase_       :    Any								=			unicodedata.category(lowercase_	)
										if cat == "Zs":
												return True
								return False



						def 					SCREAMING_SNAKE_CASE__			(				self    :				int   ,      SCREAMING_SNAKE_CASE_    :				int	):
								lowerCAmelCase_       :    Optional[int]								=			{}
								with io.open(lowercase_   ,      'r'   ,      encoding='utf-8'	) as f:
										for index, line in enumerate(lowercase_	):
												lowerCAmelCase_       :    str								=			line.rstrip('\n'	)
												lowerCAmelCase_       :    str								=			int(lowercase_	)

								return token_to_idx





						def 					SCREAMING_SNAKE_CASE__			(				self    :				int   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				Optional[str] = None	):
								lowerCAmelCase_       :    Any								=			0
								if os.path.isdir(lowercase_	):
										lowerCAmelCase_       :    Union[str, Any]								=			os.path.join(
										    lowercase_   ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
								else:
										lowerCAmelCase_       :    List[str]								=			(filename_prefix + '-' if filename_prefix else '') + save_directory
								with open(lowercase_   ,      'w'   ,      encoding='utf-8'	) as writer:
										for token, token_index in sorted(self.vocab.items()   ,      key=lambda SCREAMING_SNAKE_CASE_	: kv[1]	):
												if index != token_index:
														logger.warning(
														    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
														    ' Please check that the vocabulary is not corrupted!'	)
														lowerCAmelCase_       :    Optional[int]								=			token_index
												writer.write(token + '\n'	)
												index += 1

								lowerCAmelCase_       :    Union[str, Any]								=			os.path.join(lowercase_   ,      'sentencepiece.bpe.model'	)
								with open(lowercase_   ,      'wb'	) as fi:
										lowerCAmelCase_       :    Optional[Any]								=			self.sp_model.serialized_model_proto()
										fi.write(lowercase_	)

								return (vocab_file,)










"""simple docstring"""







import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForPreTraining,
         AlbertForQuestionAnswering,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertModel,
     )
     from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 				UpperCamelCase__       :




      """simple docstring"""




      def __init__(				self    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Dict=1_3   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=7   ,      SCREAMING_SNAKE_CASE_    :				Dict=True   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=True   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=True   ,      SCREAMING_SNAKE_CASE_    :				str=True   ,      SCREAMING_SNAKE_CASE_    :				List[str]=9_9   ,      SCREAMING_SNAKE_CASE_    :				int=1_6   ,      SCREAMING_SNAKE_CASE_    :				List[str]=3_6   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=6   ,      SCREAMING_SNAKE_CASE_    :				Tuple=6   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=6   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=3_7   ,      SCREAMING_SNAKE_CASE_    :				Tuple="gelu"   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=0.1   ,      SCREAMING_SNAKE_CASE_    :				int=0.1   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=5_1_2   ,      SCREAMING_SNAKE_CASE_    :				List[str]=1_6   ,      SCREAMING_SNAKE_CASE_    :				List[str]=2   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=0.02   ,      SCREAMING_SNAKE_CASE_    :				Dict=3   ,      SCREAMING_SNAKE_CASE_    :				int=4   ,      SCREAMING_SNAKE_CASE_    :				Tuple=None   ,      ):
        lowerCAmelCase_       :    Any								=			parent
        lowerCAmelCase_       :    Optional[int]								=			batch_size
        lowerCAmelCase_       :    Dict								=			seq_length
        lowerCAmelCase_       :    Tuple								=			is_training
        lowerCAmelCase_       :    str								=			use_input_mask
        lowerCAmelCase_       :    Union[str, Any]								=			use_token_type_ids
        lowerCAmelCase_       :    Tuple								=			use_labels
        lowerCAmelCase_       :    Optional[int]								=			vocab_size
        lowerCAmelCase_       :    Any								=			embedding_size
        lowerCAmelCase_       :    Optional[Any]								=			hidden_size
        lowerCAmelCase_       :    str								=			num_hidden_layers
        lowerCAmelCase_       :    Optional[Any]								=			num_hidden_groups
        lowerCAmelCase_       :    Dict								=			num_attention_heads
        lowerCAmelCase_       :    Optional[Any]								=			intermediate_size
        lowerCAmelCase_       :    Any								=			hidden_act
        lowerCAmelCase_       :    Union[str, Any]								=			hidden_dropout_prob
        lowerCAmelCase_       :    int								=			attention_probs_dropout_prob
        lowerCAmelCase_       :    int								=			max_position_embeddings
        lowerCAmelCase_       :    List[Any]								=			type_vocab_size
        lowerCAmelCase_       :    Any								=			type_sequence_label_size
        lowerCAmelCase_       :    Optional[int]								=			initializer_range
        lowerCAmelCase_       :    Tuple								=			num_labels
        lowerCAmelCase_       :    Dict								=			num_choices
        lowerCAmelCase_       :    Tuple								=			scope



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        lowerCAmelCase_       :    List[str]								=			ids_tensor([self.batch_size, self.seq_length]   ,      self.vocab_size	)

        lowerCAmelCase_       :    str								=			None
        if self.use_input_mask:
          lowerCAmelCase_       :    int								=			random_attention_mask([self.batch_size, self.seq_length]	)

        lowerCAmelCase_       :    List[Any]								=			None
        if self.use_token_type_ids:
          lowerCAmelCase_       :    str								=			ids_tensor([self.batch_size, self.seq_length]   ,      self.type_vocab_size	)

        lowerCAmelCase_       :    List[str]								=			None
        lowerCAmelCase_       :    Union[str, Any]								=			None
        lowerCAmelCase_       :    str								=			None
        if self.use_labels:
          lowerCAmelCase_       :    Any								=			ids_tensor([self.batch_size]   ,      self.type_sequence_label_size	)
          lowerCAmelCase_       :    List[str]								=			ids_tensor([self.batch_size, self.seq_length]   ,      self.num_labels	)
          lowerCAmelCase_       :    Dict								=			ids_tensor([self.batch_size]   ,      self.num_choices	)

        lowerCAmelCase_       :    List[Any]								=			self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        return AlbertConfig(
            vocab_size=self.vocab_size   ,      hidden_size=self.hidden_size   ,      num_hidden_layers=self.num_hidden_layers   ,      num_attention_heads=self.num_attention_heads   ,      intermediate_size=self.intermediate_size   ,      hidden_act=self.hidden_act   ,      hidden_dropout_prob=self.hidden_dropout_prob   ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,      max_position_embeddings=self.max_position_embeddings   ,      type_vocab_size=self.type_vocab_size   ,      initializer_range=self.initializer_range   ,      num_hidden_groups=self.num_hidden_groups   ,      )



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple   ,      SCREAMING_SNAKE_CASE_    :				Any   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				str	):
        lowerCAmelCase_       :    Union[str, Any]								=			AlbertModel(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    List[str]								=			model(SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    List[Any]								=			model(SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    int								=			model(SCREAMING_SNAKE_CASE_	)
        self.parent.assertEqual(result.last_hidden_state.shape   ,      (self.batch_size, self.seq_length, self.hidden_size)	)
        self.parent.assertEqual(result.pooler_output.shape   ,      (self.batch_size, self.hidden_size)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				int   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Tuple   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Tuple	):
        lowerCAmelCase_       :    Optional[Any]								=			AlbertForPreTraining(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    Optional[Any]								=			model(
            SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_   ,      labels=SCREAMING_SNAKE_CASE_   ,      sentence_order_label=SCREAMING_SNAKE_CASE_   ,      )
        self.parent.assertEqual(result.prediction_logits.shape   ,      (self.batch_size, self.seq_length, self.vocab_size)	)
        self.parent.assertEqual(result.sop_logits.shape   ,      (self.batch_size, config.num_labels)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Tuple   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				Any   ,      SCREAMING_SNAKE_CASE_    :				Dict	):
        lowerCAmelCase_       :    str								=			AlbertForMaskedLM(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    str								=			model(SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_   ,      labels=SCREAMING_SNAKE_CASE_	)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.seq_length, self.vocab_size)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				int   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				int   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]	):
        lowerCAmelCase_       :    List[str]								=			AlbertForQuestionAnswering(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    Any								=			model(
            SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_   ,      start_positions=SCREAMING_SNAKE_CASE_   ,      end_positions=SCREAMING_SNAKE_CASE_   ,      )
        self.parent.assertEqual(result.start_logits.shape   ,      (self.batch_size, self.seq_length)	)
        self.parent.assertEqual(result.end_logits.shape   ,      (self.batch_size, self.seq_length)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Dict	):
        lowerCAmelCase_       :    Union[str, Any]								=			self.num_labels
        lowerCAmelCase_       :    Union[str, Any]								=			AlbertForSequenceClassification(SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    Union[str, Any]								=			model(SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_   ,      labels=SCREAMING_SNAKE_CASE_	)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.num_labels)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Any   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				List[str]	):
        lowerCAmelCase_       :    List[str]								=			self.num_labels
        lowerCAmelCase_       :    List[Any]								=			AlbertForTokenClassification(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    str								=			model(SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_   ,      labels=SCREAMING_SNAKE_CASE_	)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.seq_length, self.num_labels)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Any   ,      SCREAMING_SNAKE_CASE_    :				str   ,      SCREAMING_SNAKE_CASE_    :				List[str]	):
        lowerCAmelCase_       :    Optional[Any]								=			self.num_choices
        lowerCAmelCase_       :    int								=			AlbertForMultipleChoice(config=SCREAMING_SNAKE_CASE_	)
        model.to(SCREAMING_SNAKE_CASE_	)
        model.eval()
        lowerCAmelCase_       :    Optional[int]								=			input_ids.unsqueeze(1	).expand(-1   ,      self.num_choices   ,      -1	).contiguous()
        lowerCAmelCase_       :    List[Any]								=			token_type_ids.unsqueeze(1	).expand(-1   ,      self.num_choices   ,      -1	).contiguous()
        lowerCAmelCase_       :    List[Any]								=			input_mask.unsqueeze(1	).expand(-1   ,      self.num_choices   ,      -1	).contiguous()
        lowerCAmelCase_       :    List[Any]								=			model(
            SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_   ,      token_type_ids=SCREAMING_SNAKE_CASE_   ,      labels=SCREAMING_SNAKE_CASE_   ,      )
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.num_choices)	)





      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]	):
        lowerCAmelCase_       :    Optional[int]								=			self.prepare_config_and_inputs()
        (
            (
            lowerCAmelCase_
        )						,(
            lowerCAmelCase_
        )						,(
            lowerCAmelCase_
        )						,(
            lowerCAmelCase_
        )						,(
            lowerCAmelCase_
        )						,(
            lowerCAmelCase_
        )						,(
            lowerCAmelCase_
        )						,
        )       :    Optional[int]								=			config_and_inputs
        lowerCAmelCase_       :    Dict								=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
        return config, inputs_dict







@require_torch
class 				UpperCamelCase__       (     lowercase_, lowercase_, unittest.TestCase  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					(
          (
              AlbertModel,
              AlbertForPreTraining,
              AlbertForMaskedLM,
              AlbertForMultipleChoice,
              AlbertForSequenceClassification,
              AlbertForTokenClassification,
              AlbertForQuestionAnswering,
          )
          if is_torch_available()
          else ()
      )
      _SCREAMING_SNAKE_CASE      =					(
          {
              """feature-extraction""": AlbertModel,
              """fill-mask""": AlbertForMaskedLM,
              """question-answering""": AlbertForQuestionAnswering,
              """text-classification""": AlbertForSequenceClassification,
              """token-classification""": AlbertForTokenClassification,
              """zero-shot""": AlbertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      _SCREAMING_SNAKE_CASE      =					True



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				List[str]   ,      SCREAMING_SNAKE_CASE_    :				str=False	):
        lowerCAmelCase_       :    List[str]								=			super()._prepare_for_class(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_   ,      return_labels=SCREAMING_SNAKE_CASE_	)

        if return_labels:
          if model_class in get_values(SCREAMING_SNAKE_CASE_	):
            lowerCAmelCase_       :    List[str]								=			torch.zeros(
                (self.model_tester.batch_size, self.model_tester.seq_length)   ,      dtype=torch.long   ,      device=SCREAMING_SNAKE_CASE_	)
            lowerCAmelCase_       :    Any								=			torch.zeros(
                self.model_tester.batch_size   ,      dtype=torch.long   ,      device=SCREAMING_SNAKE_CASE_	)
        return inputs_dict



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        lowerCAmelCase_       :    str								=			AlbertModelTester(self	)
        lowerCAmelCase_       :    Optional[int]								=			ConfigTester(self   ,      config_class=SCREAMING_SNAKE_CASE_   ,      hidden_size=3_7	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]	):
        self.config_tester.run_common_tests()



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        lowerCAmelCase_       :    List[Any]								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]	):
        lowerCAmelCase_       :    Any								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        lowerCAmelCase_       :    Tuple								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        lowerCAmelCase_       :    List[Any]								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        lowerCAmelCase_       :    str								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        lowerCAmelCase_       :    Optional[int]								=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]	):
        lowerCAmelCase_       :    Tuple								=			self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
          lowerCAmelCase_       :    int								=			type
          self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)





      @slow
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[Any]	):
        for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase_       :    Optional[Any]								=			AlbertModel.from_pretrained(SCREAMING_SNAKE_CASE_	)
          self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)







@require_torch
class 				UpperCamelCase__       (     unittest.TestCase  ):




      """simple docstring"""




      @slow
      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]	):
        lowerCAmelCase_       :    Any								=			AlbertModel.from_pretrained('albert-base-v2'	)
        lowerCAmelCase_       :    Tuple								=			torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]	)
        lowerCAmelCase_       :    Tuple								=			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
        with torch.no_grad():
          lowerCAmelCase_       :    List[Any]								=			model(SCREAMING_SNAKE_CASE_   ,      attention_mask=SCREAMING_SNAKE_CASE_	)[0]
        lowerCAmelCase_       :    str								=			torch.Size((1, 1_1, 7_6_8)	)
        self.assertEqual(output.shape   ,      SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    int								=			torch.tensor(
            [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]	)

        self.assertTrue(torch.allclose(output[:, 1:4, 1:4]   ,      SCREAMING_SNAKE_CASE_   ,      atol=1E-4	)	)





