


import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase			=				logging.get_logger(__name__)


lowerCAmelCase			=				{
    'vocab_file': 'vocab.json',
    'tokenizer_config_file': 'tokenizer_config.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase			=				{
    'vocab_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json'
        ),
    },
    'tokenizer_config_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json'
        ),
    },
    'merges_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt'
        ),
    },
}

lowerCAmelCase			=				'</w>'
lowerCAmelCase			=				'@@ '
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		lowercase__				= set()
		lowercase__				= word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)	)
				lowercase__				= char
		return pairs


# Speech2Text2 has no max input length
lowerCAmelCase			=				{'facebook/s2t-wav2vec2-large-en-de': 1024}


class 			_a				(     UpperCamelCase__     ):
							_lowercase      :     List[str]     =					VOCAB_FILES_NAMES
							_lowercase      :     List[Any]     =					PRETRAINED_VOCAB_FILES_MAP
							_lowercase      :     Optional[Any]     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_lowercase      :     int     =					['''input_ids''', '''attention_mask''']

							def __init__(				self:      Optional[int]       ,		UpperCamelCase_:      Optional[Any]       ,		UpperCamelCase_:      Optional[int]="<s>"       ,		UpperCamelCase_:      Dict="<pad>"       ,		UpperCamelCase_:      Tuple="</s>"       ,		UpperCamelCase_:      Any="<unk>"       ,		UpperCamelCase_:      Optional[int]=False       ,		UpperCamelCase_:      str=None       ,		**UpperCamelCase_:      Dict       ,		)     ->  Dict:





									"""simple docstring"""







									super().__init__(
									    unk_token=UpperCamelCase_       ,		bos_token=UpperCamelCase_       ,		eos_token=UpperCamelCase_       ,		pad_token=UpperCamelCase_       ,		do_lower_case=UpperCamelCase_       ,		**UpperCamelCase_       ,		)

									lowercase__				= do_lower_case

									with open(UpperCamelCase_       ,		encoding='''utf-8'''				) as vocab_handle:
											lowercase__				= json.load(UpperCamelCase_				)
									lowercase__				= {v: k for k, v in self.encoder.items()}

									if merges_file is None:
											logger.info(f'No merges files provided. {self.__class__.__name__} can only be used for decoding.'				)

											lowercase__				= None
											lowercase__				= None
									else:
											with open(UpperCamelCase_       ,		encoding='''utf-8'''				) as merges_handle:
													lowercase__				= merges_handle.read().split('''\n'''				)[:-1]

											lowercase__				= [tuple(merge.split()[:2]				) for merge in merges]
											lowercase__				= dict(zip(UpperCamelCase_       ,		range(len(UpperCamelCase_				)				)				)				)
											lowercase__				= {}

							@property
							def  lowerCamelCase_					(				self:      str				)     ->  int:





									"""simple docstring"""







									return len(self.decoder				)

							def  lowerCamelCase_					(				self:      Union[str, Any]				)     ->  Dict:





									"""simple docstring"""







									return dict(self.encoder       ,		**self.added_tokens_encoder				)

							def  lowerCamelCase_					(				self:      Dict       ,		UpperCamelCase_:      Optional[int]				)     ->  str:





									"""simple docstring"""







									lowercase__				= tuple(token[:-1]				) + (token[-1] + BPE_TOKEN_MERGES,)
									if token in self.cache:
											return self.cache[token]
									lowercase__				= get_pairs(UpperCamelCase_				)

									if not pairs:
											return token

									while True:
											lowercase__				= min(UpperCamelCase_       ,		key=lambda UpperCamelCase_				: self.bpe_ranks.get(UpperCamelCase_       ,		float('''inf'''				)				)				)
											if bigram not in self.bpe_ranks:
													break
											lowercase__	,						lowercase__				= bigram
											lowercase__				= []
											lowercase__				= 0
											while i < len(UpperCamelCase_				):
													try:
															lowercase__				= word.index(UpperCamelCase_       ,		UpperCamelCase_				)
													except ValueError:
															new_word.extend(word[i:]				)
															break
													else:
															new_word.extend(word[i:j]				)
															lowercase__				= j

													if word[i] == first and i < len(UpperCamelCase_				) - 1 and word[i + 1] == second:
															new_word.append(first + second				)
															i += 2
													else:
															new_word.append(word[i]				)
															i += 1
											lowercase__				= tuple(UpperCamelCase_				)
											lowercase__				= new_word
											if len(UpperCamelCase_				) == 1:
													break
											else:
													lowercase__				= get_pairs(UpperCamelCase_				)
									lowercase__				= ''' '''.join(UpperCamelCase_				)
									if word == "\n  " + BPE_TOKEN_MERGES:
											lowercase__				= '''\n''' + BPE_TOKEN_MERGES

									if word.endswith(UpperCamelCase_				):
											lowercase__				= word.replace(UpperCamelCase_       ,		''''''				)

									lowercase__				= word.replace(''' '''       ,		UpperCamelCase_				)
									lowercase__				= word
									return word

							def  lowerCamelCase_					(				self:      str       ,		UpperCamelCase_:      str				)     ->  Optional[int]:





									"""simple docstring"""







									if self.bpe_ranks is None:
											raise ValueError(
											    '''This tokenizer was instantiated without a `merges.txt` file, so'''
											    ''' that it can only be used for decoding, not for encoding.'''
											    '''Make sure to provide `merges.txt` file at instantiation to enable '''
											    '''encoding.'''				)

									if self.do_lower_case:
											lowercase__				= text.lower()

									lowercase__				= text.split()

									lowercase__				= []
									for token in text:
											if token:
													split_tokens.extend(list(self.bpe(UpperCamelCase_				).split(''' '''				)				)				)

									return split_tokens

							def  lowerCamelCase_					(				self:      Dict       ,		UpperCamelCase_:      str				)     ->  int:





									"""simple docstring"""







									return self.encoder.get(UpperCamelCase_       ,		self.encoder.get(self.unk_token				)				)

							def  lowerCamelCase_					(				self:      Optional[Any]       ,		UpperCamelCase_:      int				)     ->  str:





									"""simple docstring"""







									lowercase__				= self.decoder.get(UpperCamelCase_       ,		self.unk_token				)
									return result

							def  lowerCamelCase_					(				self:      int       ,		UpperCamelCase_:      List[str]				)     ->  str:





									"""simple docstring"""







									lowercase__				= ''' '''.join(UpperCamelCase_				)

									# make sure @@ tokens are concatenated
									lowercase__				= ''''''.join(string.split(UpperCamelCase_				)				)

									return string


							def  lowerCamelCase_					(				self:      str       ,		UpperCamelCase_:      str       ,		UpperCamelCase_:      Optional[str] = None				)     ->  Tuple[str]:





									"""simple docstring"""







									if not os.path.isdir(UpperCamelCase_				):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'				)
											return
									lowercase__				= os.path.join(
									    UpperCamelCase_       ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
									lowercase__				= os.path.join(
									    UpperCamelCase_       ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

									with open(UpperCamelCase_       ,		'''w'''       ,		encoding='''utf-8'''				) as f:
											f.write(json.dumps(self.encoder       ,		indent=2       ,		sort_keys=UpperCamelCase_       ,		ensure_ascii=UpperCamelCase_				) + '''\n'''				)

									lowercase__				= 0
									if self.bpe_ranks is None:
											return (vocab_file,)

									with open(UpperCamelCase_       ,		'''w'''       ,		encoding='''utf-8'''				) as writer:
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,		key=lambda UpperCamelCase_				: kv[1]				):
													if index != token_index:
															logger.warning(
															    f'Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'
															    ''' Please check that the tokenizer is not corrupted!'''				)
															lowercase__				= token_index
													writer.write(''' '''.join(UpperCamelCase_				) + '''\n'''				)
													index += 1

									return (vocab_file, merges_file)




"""simple docstring"""
import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)

def  lowercase     (_lowerCAmelCase						):
						__lowerCAmelCase     =  torch.exp(_lowerCAmelCase						)
						__lowerCAmelCase     =  torch.sum(_lowerCAmelCase ,    dim=1						)  # sum of exp(x_i)
						__lowerCAmelCase     =  torch.sum(x * exp_x ,    dim=1						)  # sum of x_i * exp(x_i)
						return torch.log(_lowerCAmelCase						) - B / A
class       lowerCAmelCase_		(				nn.Module ):



							'''simple docstring'''





							def __init__( self    ,				snake_case_				)       ->  Optional[Any]:
													super().__init__()
													__lowerCAmelCase     =  config.output_attentions
													__lowerCAmelCase     =  config.output_hidden_states
													__lowerCAmelCase     =  nn.ModuleList([BertLayer(snake_case_				) for _ in range(config.num_hidden_layers				)]				)
													__lowerCAmelCase     =  nn.ModuleList([BertHighway(snake_case_				) for _ in range(config.num_hidden_layers				)]				)

													__lowerCAmelCase     =  [-1 for _ in range(config.num_hidden_layers				)]




							def    A__						( self    ,				snake_case_				)       ->  List[Any]:
													if (type(snake_case_				) is float) or (type(snake_case_				) is int):
																			for i in range(len(self.early_exit_entropy				)				):
																									__lowerCAmelCase     =  x
													else:
																			__lowerCAmelCase     =  x




							def    A__						( self    ,				snake_case_				)       ->  Dict:
													__lowerCAmelCase     =  pooler.state_dict()
													for highway in self.highway:
																			for name, param in highway.pooler.state_dict().items():
																									param.copy_(loaded_model[name]				)




							def    A__						( self    ,				snake_case_    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				)       ->  List[str]:
													__lowerCAmelCase     =  ()
													__lowerCAmelCase     =  ()
													__lowerCAmelCase     =  ()
													for i, layer_module in enumerate(self.layer				):
																			if self.output_hidden_states:
																									__lowerCAmelCase     =  all_hidden_states + (hidden_states,)

																			__lowerCAmelCase     =  layer_module(
																			    snake_case_    ,				snake_case_    ,				head_mask[i]    ,				snake_case_    ,				snake_case_				)
																			__lowerCAmelCase     =  layer_outputs[0]

																			if self.output_attentions:
																									__lowerCAmelCase     =  all_attentions + (layer_outputs[1],)

																			__lowerCAmelCase     =  (hidden_states,)
																			if self.output_hidden_states:
																									__lowerCAmelCase     =  current_outputs + (all_hidden_states,)
																			if self.output_attentions:
																									__lowerCAmelCase     =  current_outputs + (all_attentions,)

																			__lowerCAmelCase     =  self.highway[i](snake_case_				)
																			# logits, pooled_output

																			if not self.training:
																									__lowerCAmelCase     =  highway_exit[0]
																									__lowerCAmelCase     =  entropy(snake_case_				)
																									__lowerCAmelCase     =  highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
																									__lowerCAmelCase     =  all_highway_exits + (highway_exit,)

																									if highway_entropy < self.early_exit_entropy[i]:
																															__lowerCAmelCase     =  (highway_logits,) + current_outputs[1:] + (all_highway_exits,)
																															raise HighwayException(snake_case_    ,				i + 1				)
																			else:
																									__lowerCAmelCase     =  all_highway_exits + (highway_exit,)

        # Add last layer
													if self.output_hidden_states:
																			__lowerCAmelCase     =  all_hidden_states + (hidden_states,)

													__lowerCAmelCase     =  (hidden_states,)
													if self.output_hidden_states:
																			__lowerCAmelCase     =  outputs + (all_hidden_states,)
													if self.output_attentions:
																			__lowerCAmelCase     =  outputs + (all_attentions,)

													__lowerCAmelCase     =  outputs + (all_highway_exits,)
													return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits
@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''				,				A__				,				)
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							def __init__( self    ,				snake_case_				)       ->  Tuple:
													super().__init__(snake_case_				)
													__lowerCAmelCase     =  config

													__lowerCAmelCase     =  BertEmbeddings(snake_case_				)
													__lowerCAmelCase     =  DeeBertEncoder(snake_case_				)
													__lowerCAmelCase     =  BertPooler(snake_case_				)

													self.init_weights()




							def    A__						( self				)       ->  Dict:
													self.encoder.init_highway_pooler(self.pooler				)




							def    A__						( self				)       ->  List[Any]:
													return self.embeddings.word_embeddings




							def    A__						( self    ,				snake_case_				)       ->  Tuple:
													__lowerCAmelCase     =  value




							def    A__						( self    ,				snake_case_				)       ->  List[str]:
													for layer, heads in heads_to_prune.items():
																			self.encoder.layer[layer].attention.prune_heads(snake_case_				)




							@add_start_docstrings_to_model_forward(snake_case_				)
							def    A__						( self    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				)       ->  Optional[Any]:
													if input_ids is not None and inputs_embeds is not None:
																			raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""				)
													elif input_ids is not None:
																			__lowerCAmelCase     =  input_ids.size()
													elif inputs_embeds is not None:
																			__lowerCAmelCase     =  inputs_embeds.size()[:-1]
													else:
																			raise ValueError("""You have to specify either input_ids or inputs_embeds"""				)

													__lowerCAmelCase     =  input_ids.device if input_ids is not None else inputs_embeds.device

													if attention_mask is None:
																			__lowerCAmelCase     =  torch.ones(snake_case_    ,				device=snake_case_				)
													if encoder_attention_mask is None:
																			__lowerCAmelCase     =  torch.ones(snake_case_    ,				device=snake_case_				)
													if token_type_ids is None:
																			__lowerCAmelCase     =  torch.zeros(snake_case_    ,				dtype=torch.long    ,				device=snake_case_				)

													# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
													# ourselves in which case we just need to make it broadcastable to all heads.
													__lowerCAmelCase     =  self.get_extended_attention_mask(snake_case_    ,				snake_case_    ,				snake_case_				)

													# If a 2D ou 3D attention mask is provided for the cross-attention
													# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
													if encoder_attention_mask.dim() == 3:
																			__lowerCAmelCase     =  encoder_attention_mask[:, None, :, :]
													if encoder_attention_mask.dim() == 2:
																			__lowerCAmelCase     =  encoder_attention_mask[:, None, None, :]

													__lowerCAmelCase     =  encoder_extended_attention_mask.to(
													    dtype=next(self.parameters()				).dtype				)  # fp16 compatibility
													__lowerCAmelCase     =  (1.0 - encoder_extended_attention_mask) * -10_000.0

													# Prepare head mask if needed
													# 1.0 in head_mask indicate we keep the head
													# attention_probs has shape bsz x n_heads x N x N
													# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
													# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
													__lowerCAmelCase     =  self.get_head_mask(snake_case_    ,				self.config.num_hidden_layers				)

													__lowerCAmelCase     =  self.embeddings(
													    input_ids=snake_case_    ,				position_ids=snake_case_    ,				token_type_ids=snake_case_    ,				inputs_embeds=snake_case_				)
													__lowerCAmelCase     =  self.encoder(
													    snake_case_    ,				attention_mask=snake_case_    ,				head_mask=snake_case_    ,				encoder_hidden_states=snake_case_    ,				encoder_attention_mask=snake_case_    ,				)
													__lowerCAmelCase     =  encoder_outputs[0]
													__lowerCAmelCase     =  self.pooler(snake_case_				)

													__lowerCAmelCase     =  (
													    sequence_output,
													    pooled_output,
													) + encoder_outputs[
													    1:
													]  # add hidden_states and attentions if they are here
													return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							def __init__( self    ,				snake_case_    ,				snake_case_				)       ->  Optional[int]:
													__lowerCAmelCase     =  message
													__lowerCAmelCase     =  exit_layer  # start from 1!
class       lowerCAmelCase_		(				nn.Module ):



							'''simple docstring'''





							def __init__( self    ,				snake_case_				)       ->  List[Any]:
													super().__init__()
													__lowerCAmelCase     =  BertPooler(snake_case_				)
													__lowerCAmelCase     =  nn.Dropout(config.hidden_dropout_prob				)
													__lowerCAmelCase     =  nn.Linear(config.hidden_size    ,				config.num_labels				)




							def    A__						( self    ,				snake_case_				)       ->  Optional[int]:
													# Pooler
													__lowerCAmelCase     =  encoder_outputs[0]
													__lowerCAmelCase     =  self.pooler(snake_case_				)
													# "return" pooler_output

													# BertModel
													__lowerCAmelCase     =  (pooler_input, pooler_output) + encoder_outputs[1:]
													# "return" bmodel_output

													# Dropout and classification
													__lowerCAmelCase     =  bmodel_output[1]

													__lowerCAmelCase     =  self.dropout(snake_case_				)
													__lowerCAmelCase     =  self.classifier(snake_case_				)

													return logits, pooled_output
@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''				,				A__				,				)
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							def __init__( self    ,				snake_case_				)       ->  Optional[Any]:
													super().__init__(snake_case_				)
													__lowerCAmelCase     =  config.num_labels
													__lowerCAmelCase     =  config.num_hidden_layers

													__lowerCAmelCase     =  DeeBertModel(snake_case_				)
													__lowerCAmelCase     =  nn.Dropout(config.hidden_dropout_prob				)
													__lowerCAmelCase     =  nn.Linear(config.hidden_size    ,				self.config.num_labels				)

													self.init_weights()




							@add_start_docstrings_to_model_forward(snake_case_				)
							def    A__						( self    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=-1    ,				snake_case_=False    ,				)       ->  Optional[int]:
													__lowerCAmelCase     =  self.num_layers
													try:
																			__lowerCAmelCase     =  self.bert(
																			    snake_case_    ,				attention_mask=snake_case_    ,				token_type_ids=snake_case_    ,				position_ids=snake_case_    ,				head_mask=snake_case_    ,				inputs_embeds=snake_case_    ,				)
																			# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

																			__lowerCAmelCase     =  outputs[1]

																			__lowerCAmelCase     =  self.dropout(snake_case_				)
																			__lowerCAmelCase     =  self.classifier(snake_case_				)
																			__lowerCAmelCase     =  (logits,) + outputs[2:]  # add hidden states and attention if they are here
													except HighwayException as e:
																			__lowerCAmelCase     =  e.message
																			__lowerCAmelCase     =  e.exit_layer
																			__lowerCAmelCase     =  outputs[0]

													if not self.training:
																			__lowerCAmelCase     =  entropy(snake_case_				)
																			__lowerCAmelCase     =  []
																			__lowerCAmelCase     =  []
													if labels is not None:
																			if self.num_labels == 1:
																									#  We are doing regression
																									__lowerCAmelCase     =  MSELoss()
																									__lowerCAmelCase     =  loss_fct(logits.view(-1				)    ,				labels.view(-1				)				)
																			else:
																									__lowerCAmelCase     =  CrossEntropyLoss()
																									__lowerCAmelCase     =  loss_fct(logits.view(-1    ,				self.num_labels				)    ,				labels.view(-1				)				)

																			# work with highway exits
																			__lowerCAmelCase     =  []
																			for highway_exit in outputs[-1]:
																									__lowerCAmelCase     =  highway_exit[0]
																									if not self.training:
																															highway_logits_all.append(snake_case_				)
																															highway_entropy.append(highway_exit[2]				)
																									if self.num_labels == 1:
																															#  We are doing regression
																															__lowerCAmelCase     =  MSELoss()
																															__lowerCAmelCase     =  loss_fct(highway_logits.view(-1				)    ,				labels.view(-1				)				)
																									else:
																															__lowerCAmelCase     =  CrossEntropyLoss()
																															__lowerCAmelCase     =  loss_fct(highway_logits.view(-1    ,				self.num_labels				)    ,				labels.view(-1				)				)
																									highway_losses.append(snake_case_				)

																			if train_highway:
																									__lowerCAmelCase     =  (sum(highway_losses[:-1]				),) + outputs
																									# exclude the final highway, of course
																			else:
																									__lowerCAmelCase     =  (loss,) + outputs
													if not self.training:
																			__lowerCAmelCase     =  outputs + ((original_entropy, highway_entropy), exit_layer)
																			if output_layer >= 0:
																									__lowerCAmelCase     =  (
																									    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																									)  # use the highway of the last layer

													return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)
