




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


a_      						=   logging.getLogger(__name__)

class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      def 					snake_case		(   self    :       Optional[int]       ,				snake_case    :       Dict       ,				snake_case    :       Union[str, Any]       ,				snake_case    :       List[str]=None       ,				snake_case    :       List[str]=None ):
          if current_layer == 0:
              __UpperCamelCase     						=  self.embedding_hidden_mapping_in(snake_case )
          else:
              __UpperCamelCase     						=  hidden_states[0]

          __UpperCamelCase     						=  int(self.config.num_hidden_layers / self.config.num_hidden_groups )

          # Index of the hidden group
          __UpperCamelCase     						=  int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups) )

          __UpperCamelCase     						=  self.albert_layer_groups[group_idx](
              snake_case       ,				snake_case       ,				head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]       ,				)
          __UpperCamelCase     						=  layer_group_output[0]

          return (hidden_states,)







@add_start_docstrings(
    "The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."							,	_SCREAMING_SNAKE_CASE							,	)
class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      def __init__(   self    :       Any       ,				snake_case    :       List[Any] ):
          super().__init__(snake_case )

          __UpperCamelCase     						=  AlbertTransformerWithPabee(snake_case )

          self.init_weights()
          __UpperCamelCase     						=  0
          __UpperCamelCase     						=  0
          __UpperCamelCase     						=  0

          __UpperCamelCase     						=  0







      def 					snake_case		(   self    :       List[Any]       ,				snake_case    :       int ):
          __UpperCamelCase     						=  threshold







      def 					snake_case		(   self    :       Any       ,				snake_case    :       str ):
          __UpperCamelCase     						=  patience







      def 					snake_case		(   self    :       Dict ):
          __UpperCamelCase     						=  0
          __UpperCamelCase     						=  0







      def 					snake_case		(   self    :       Any ):
          __UpperCamelCase     						=  self.inference_layers_num / self.inference_instances_num
          __UpperCamelCase     						=  (
              F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
              F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
          )
          print(snake_case )







      @add_start_docstrings_to_model_forward(snake_case )
      def 					snake_case		(   self    :       Tuple       ,				snake_case    :       List[str]=None       ,				snake_case    :       Dict=None       ,				snake_case    :       Optional[Any]=None       ,				snake_case    :       List[str]=None       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       str=None       ,				snake_case    :       Any=None       ,				snake_case    :       Optional[Any]=None       ,				snake_case    :       List[Any]=False       ,				):

          if input_ids is not None and inputs_embeds is not None:
              raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time''' )
          elif input_ids is not None:
              __UpperCamelCase     						=  input_ids.size()
          elif inputs_embeds is not None:
              __UpperCamelCase     						=  inputs_embeds.size()[:-1]
          else:
              raise ValueError('''You have to specify either input_ids or inputs_embeds''' )

          __UpperCamelCase     						=  input_ids.device if input_ids is not None else inputs_embeds.device

          if attention_mask is None:
              __UpperCamelCase     						=  torch.ones(snake_case       ,				device=snake_case )
          if token_type_ids is None:
              __UpperCamelCase     						=  torch.zeros(snake_case       ,				dtype=torch.long       ,				device=snake_case )

          __UpperCamelCase     						=  attention_mask.unsqueeze(1 ).unsqueeze(2 )
          __UpperCamelCase     						=  extended_attention_mask.to(dtype=self.dtype )  # fp16 compatibility
          __UpperCamelCase     						=  (1.0 - extended_attention_mask) * -1_00_00.0
          __UpperCamelCase     						=  self.get_head_mask(snake_case       ,				self.config.num_hidden_layers )

          __UpperCamelCase     						=  self.embeddings(
              snake_case       ,				position_ids=snake_case       ,				token_type_ids=snake_case       ,				inputs_embeds=snake_case )
          __UpperCamelCase     						=  embedding_output

          if self.training:
              __UpperCamelCase     						=  []
              for i in range(self.config.num_hidden_layers ):
                  __UpperCamelCase     						=  self.encoder.adaptive_forward(
                      snake_case       ,				current_layer=snake_case       ,				attention_mask=snake_case       ,				head_mask=snake_case       ,				)

                  __UpperCamelCase     						=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0] ) )
                  __UpperCamelCase     						=  output_layers[i](output_dropout(snake_case ) )
                  res.append(snake_case )
          elif self.patience == 0:  # Use all layers for inference
              __UpperCamelCase     						=  self.encoder(snake_case       ,				snake_case       ,				head_mask=snake_case )
              __UpperCamelCase     						=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0] ) )
              __UpperCamelCase     						=  [output_layers[self.config.num_hidden_layers - 1](snake_case )]
          else:
              __UpperCamelCase     						=  0
              __UpperCamelCase     						=  None
              __UpperCamelCase     						=  0
              for i in range(self.config.num_hidden_layers ):
                  calculated_layer_num += 1
                  __UpperCamelCase     						=  self.encoder.adaptive_forward(
                      snake_case       ,				current_layer=snake_case       ,				attention_mask=snake_case       ,				head_mask=snake_case       ,				)

                  __UpperCamelCase     						=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0] ) )
                  __UpperCamelCase     						=  output_layers[i](snake_case )
                  if regression:
                      __UpperCamelCase     						=  logits.detach()
                      if patient_result is not None:
                          __UpperCamelCase     						=  patient_result.detach()
                      if (patient_result is not None) and torch.abs(patient_result - labels ) < self.regression_threshold:
                          patient_counter += 1
                      else:
                          __UpperCamelCase     						=  0
                  else:
                      __UpperCamelCase     						=  logits.detach().argmax(dim=1 )
                      if patient_result is not None:
                          __UpperCamelCase     						=  patient_result.detach().argmax(dim=1 )
                      if (patient_result is not None) and torch.all(labels.eq(snake_case ) ):
                          patient_counter += 1
                      else:
                          __UpperCamelCase     						=  0

                  __UpperCamelCase     						=  logits
                  if patient_counter == self.patience:
                      break
              __UpperCamelCase     						=  [patient_result]
              self.inference_layers_num += calculated_layer_num
              self.inference_instances_num += 1

          return res







@add_start_docstrings(
    "Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "							,	_SCREAMING_SNAKE_CASE							,	)
class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      def __init__(   self    :       Optional[int]       ,				snake_case    :       int ):
          super().__init__(snake_case )
          __UpperCamelCase     						=  config.num_labels

          __UpperCamelCase     						=  AlbertModelWithPabee(snake_case )
          __UpperCamelCase     						=  nn.Dropout(config.classifier_dropout_prob )
          __UpperCamelCase     						=  nn.ModuleList(
              [nn.Linear(config.hidden_size       ,				self.config.num_labels ) for _ in range(config.num_hidden_layers )] )

          self.init_weights()







      @add_start_docstrings_to_model_forward(snake_case )
      def 					snake_case		(   self    :       int       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       Any=None       ,				snake_case    :       Tuple=None       ,				snake_case    :       List[str]=None       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       str=None       ,				snake_case    :       List[Any]=None       ,				):
          __UpperCamelCase     						=  self.albert(
              input_ids=snake_case       ,				attention_mask=snake_case       ,				token_type_ids=snake_case       ,				position_ids=snake_case       ,				head_mask=snake_case       ,				inputs_embeds=snake_case       ,				output_dropout=self.dropout       ,				output_layers=self.classifiers       ,				regression=self.num_labels == 1       ,				)

          __UpperCamelCase     						=  (logits[-1],)

          if labels is not None:
              __UpperCamelCase     						=  None
              __UpperCamelCase     						=  0
              for ix, logits_item in enumerate(snake_case ):
                  if self.num_labels == 1:
                      #  We are doing regression
                      __UpperCamelCase     						=  MSELoss()
                      __UpperCamelCase     						=  loss_fct(logits_item.view(-1 )       ,				labels.view(-1 ) )
                  else:
                      __UpperCamelCase     						=  CrossEntropyLoss()
                      __UpperCamelCase     						=  loss_fct(logits_item.view(-1       ,				self.num_labels )       ,				labels.view(-1 ) )
                  if total_loss is None:
                      __UpperCamelCase     						=  loss
                  else:
                      total_loss += loss * (ix + 1)
                  total_weights += ix + 1
              __UpperCamelCase     						=  (total_loss / total_weights,) + outputs

          return outputs












'''simple docstring'''


import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_reformer import ReformerTokenizer
else:
				_snake_case      :      int   							=		None


_snake_case      :      Union[str, Any]   							=		logging.get_logger(__name__)


_snake_case      :      Union[str, Any]   							=		'▁'

_snake_case      :      Any   							=		{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

_snake_case      :      List[str]   							=		{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    },
    'tokenizer_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'
        )
    },
}

_snake_case      :      Union[str, Any]   							=		{
    'google/reformer-crime-and-punishment': 524288,
}

class 			A    (		_a    ):
							lowercase_							=				VOCAB_FILES_NAMES
							lowercase_							=				PRETRAINED_VOCAB_FILES_MAP
							lowercase_							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_							=				['input_ids', 'attention_mask']
							lowercase_							=				ReformerTokenizer







							def __init__(  self       :  Any							,			lowerCAmelCase_       :  int=None							,			lowerCAmelCase_       :  Optional[Any]=None							,			lowerCAmelCase_       :  List[str]="</s>"							,			lowerCAmelCase_       :  Any="<unk>"							,			lowerCAmelCase_       :  List[str]=[]							,			**lowerCAmelCase_       :  List[Any]							,			)			->							Dict:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,			tokenizer_file=lowerCAmelCase_							,			eos_token=lowerCAmelCase_							,			unk_token=lowerCAmelCase_							,			additional_special_tokens=lowerCAmelCase_							,			**lowerCAmelCase_							,			)

														_a	     =				vocab_file
														_a	     =				False if not self.vocab_file else True







							def    __lowerCAmelCase			(  self       :  List[Any]							,			lowerCAmelCase_       :  str							,			lowerCAmelCase_       :  Optional[str] = None					)			->							Tuple[str]:


														"""simple docstring"""


														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																					    '''tokenizer.'''					)

														if not os.path.isdir(lowerCAmelCase_					):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
																					return
														_a	     =				os.path.join(
														    lowerCAmelCase_							,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

														if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase_					):
																					copyfile(self.vocab_file							,			lowerCAmelCase_					)

														return (out_vocab_file,)

