'''simple docstring'''






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


__A	:	Union[str, Any]     =     pytest.mark.integration
@require_faiss
class 						__UpperCamelCase	(						lowercase__							):





							def 		a__   (  self		:Dict			):
									snake_case_		:							List[str]			       =						Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(_UpperCamelCase			) for x in np.arange(3_0			).tolist()]}			)
									return dset





							def 		a__   (  self		:Dict			):
									import faiss

									snake_case_		:							Dataset			       =						self._create_dummy_dataset()
									snake_case_		:							int			       =						dset.map(
									    lambda _UpperCamelCase  ,_UpperCamelCase			: {"vecs": i * np.ones(5  ,dtype=np.floataa			)}  ,with_indices=_UpperCamelCase  ,keep_in_memory=_UpperCamelCase			)
									snake_case_		:							Any			       =						dset.add_faiss_index("""vecs"""  ,batch_size=1_0_0  ,metric_type=faiss.METRIC_INNER_PRODUCT			)
									snake_case_		:							Dict			       =						dset.get_nearest_examples("""vecs"""  ,np.ones(5  ,dtype=np.floataa			)			)
									self.assertEqual(examples["""filename"""][0]  ,"""my_name-train_29"""			)
									dset.drop_index("""vecs"""			)





							def 		a__   (  self		:Optional[int]			):
									import faiss

									snake_case_		:							Dataset			       =						self._create_dummy_dataset()
									dset.add_faiss_index_from_external_arrays(
									    external_arrays=np.ones((3_0, 5)			) * np.arange(3_0			).reshape(-1  ,1			)  ,index_name="""vecs"""  ,batch_size=1_0_0  ,metric_type=faiss.METRIC_INNER_PRODUCT  ,)
									snake_case_		:							Union[str, Any]			       =						dset.get_nearest_examples("""vecs"""  ,np.ones(5  ,dtype=np.floataa			)			)
									self.assertEqual(examples["""filename"""][0]  ,"""my_name-train_29"""			)





							def 		a__   (  self		:Dict			):
									import faiss

									snake_case_		:							Dataset			       =						self._create_dummy_dataset()
									dset.add_faiss_index_from_external_arrays(
									    external_arrays=np.ones((3_0, 5)			) * np.arange(3_0			).reshape(-1  ,1			)  ,index_name="""vecs"""  ,metric_type=faiss.METRIC_INNER_PRODUCT  ,)

									# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
									# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
									# see https://bugs.python.org/issue14243 and
									# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
									with tempfile.NamedTemporaryFile(delete=_UpperCamelCase			) as tmp_file:
											dset.save_faiss_index("""vecs"""  ,tmp_file.name			)
											dset.load_faiss_index("""vecs2"""  ,tmp_file.name			)
									os.unlink(tmp_file.name			)

									snake_case_		:							List[Any]			       =						dset.get_nearest_examples("""vecs2"""  ,np.ones(5  ,dtype=np.floataa			)			)
									self.assertEqual(examples["""filename"""][0]  ,"""my_name-train_29"""			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							Dataset			       =						self._create_dummy_dataset()
									dset.add_faiss_index_from_external_arrays(
									    external_arrays=np.ones((3_0, 5)			) * np.arange(3_0			).reshape(-1  ,1			)  ,index_name="""vecs"""			)
									dset.drop_index("""vecs"""			)
									self.assertRaises(_UpperCamelCase  ,partial(dset.get_nearest_examples  ,"""vecs2"""  ,np.ones(5  ,dtype=np.floataa			)			)			)





							def 		a__   (  self		:Optional[Any]			):
									from elasticsearch import Elasticsearch

									snake_case_		:							Dataset			       =						self._create_dummy_dataset()
									with patch("""elasticsearch.Elasticsearch.search"""			) as mocked_search, patch(
									    """elasticsearch.client.IndicesClient.create"""			) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""			) as mocked_bulk:
											snake_case_		:							List[str]			       =						{"""acknowledged""": True}
											mocked_bulk.return_value([(True, None)] * 3_0			)
											snake_case_		:							Optional[int]			       =						{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 2_9}]}}
											snake_case_		:							Union[str, Any]			       =						Elasticsearch()

											dset.add_elasticsearch_index("""filename"""  ,es_client=_UpperCamelCase			)
											snake_case_		:							int			       =						dset.get_nearest_examples("""filename"""  ,"""my_name-train_29"""			)
											self.assertEqual(examples["""filename"""][0]  ,"""my_name-train_29"""			)





@require_faiss
class 						__UpperCamelCase	(						lowercase__							):





							def 		a__   (  self		:Union[str, Any]			):
									import faiss

									snake_case_		:							Dict			       =						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)

									# add vectors
									index.add_vectors(np.eye(5  ,dtype=np.floataa			)			)
									self.assertIsNotNone(index.faiss_index			)
									self.assertEqual(index.faiss_index.ntotal  ,5			)
									index.add_vectors(np.zeros((5, 5)  ,dtype=np.floataa			)			)
									self.assertEqual(index.faiss_index.ntotal  ,1_0			)

									# single query
									snake_case_		:							Union[str, Any]			       =						np.zeros(5  ,dtype=np.floataa			)
									snake_case_		:							Union[str, Any]			       =						1
									snake_case_		:							List[Any]			       =						index.search(_UpperCamelCase			)
									self.assertRaises(_UpperCamelCase  ,index.search  ,query.reshape(-1  ,1			)			)
									self.assertGreater(scores[0]  ,0			)
									self.assertEqual(indices[0]  ,1			)

									# batched queries
									snake_case_		:							str			       =						np.eye(5  ,dtype=np.floataa			)[::-1]
									snake_case_		:							Any			       =						index.search_batch(_UpperCamelCase			)
									self.assertRaises(_UpperCamelCase  ,index.search_batch  ,queries[0]			)
									snake_case_		:							List[str]			       =						[scores[0] for scores in total_scores]
									snake_case_		:							int			       =						[indices[0] for indices in total_indices]
									self.assertGreater(np.min(_UpperCamelCase			)  ,0			)
									self.assertListEqual([4, 3, 2, 1, 0]  ,_UpperCamelCase			)





							def 		a__   (  self		:Optional[Any]			):
									import faiss

									snake_case_		:							List[Any]			       =						FaissIndex(string_factory="""Flat"""			)
									index.add_vectors(np.eye(5  ,dtype=np.floataa			)			)
									self.assertIsInstance(index.faiss_index  ,faiss.IndexFlat			)
									snake_case_		:							Optional[Any]			       =						FaissIndex(string_factory="""LSH"""			)
									index.add_vectors(np.eye(5  ,dtype=np.floataa			)			)
									self.assertIsInstance(index.faiss_index  ,faiss.IndexLSH			)
									with self.assertRaises(_UpperCamelCase			):
											snake_case_		:							int			       =						FaissIndex(string_factory="""Flat"""  ,custom_index=faiss.IndexFlat(5			)			)





							def 		a__   (  self		:str			):
									import faiss

									snake_case_		:							Dict			       =						faiss.IndexFlat(5			)
									snake_case_		:							Dict			       =						FaissIndex(custom_index=_UpperCamelCase			)
									index.add_vectors(np.eye(5  ,dtype=np.floataa			)			)
									self.assertIsInstance(index.faiss_index  ,faiss.IndexFlat			)





							def 		a__   (  self		:Dict			):
									import faiss

									snake_case_		:							Tuple			       =						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
									index.add_vectors(np.eye(5  ,dtype=np.floataa			)			)

									# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
									# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
									# see https://bugs.python.org/issue14243 and
									# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
									with tempfile.NamedTemporaryFile(delete=_UpperCamelCase			) as tmp_file:
											index.save(tmp_file.name			)
											snake_case_		:							Union[str, Any]			       =						FaissIndex.load(tmp_file.name			)
									os.unlink(tmp_file.name			)

									snake_case_		:							Dict			       =						np.zeros(5  ,dtype=np.floataa			)
									snake_case_		:							str			       =						1
									snake_case_		:							List[Any]			       =						index.search(_UpperCamelCase			)
									self.assertGreater(scores[0]  ,0			)
									self.assertEqual(indices[0]  ,1			)





@require_faiss
def      UpperCAmelCase      (		lowerCamelCase_  :int ):







		'''simple docstring'''
		import faiss

		snake_case_		:							Dict			       =						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT )
		index.add_vectors(np.eye(5				,      dtype=np.floataa ) )

		snake_case_		:							Optional[int]			       =						"""index.faiss"""
		snake_case_		:							Optional[Any]			       =						F'''mock://{index_name}'''
		index.save(lowerCamelCase_				,      storage_options=mockfs.storage_options )
		snake_case_		:							str			       =						FaissIndex.load(lowerCamelCase_				,      storage_options=mockfs.storage_options )

		snake_case_		:							Any			       =						np.zeros(5				,      dtype=np.floataa )
		snake_case_		:							str			       =						1
		snake_case_		:							Tuple			       =						index.search(lowerCamelCase_ )
		assert scores[0] > 0
		assert indices[0] == 1






@require_elasticsearch
class 						__UpperCamelCase	(						lowercase__							):





							def 		a__   (  self		:Dict			):
									from elasticsearch import Elasticsearch

									with patch("""elasticsearch.Elasticsearch.search"""			) as mocked_search, patch(
									    """elasticsearch.client.IndicesClient.create"""			) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""			) as mocked_bulk:
											snake_case_		:							Any			       =						Elasticsearch()
											snake_case_		:							Optional[Any]			       =						{"""acknowledged""": True}
											snake_case_		:							Union[str, Any]			       =						ElasticSearchIndex(es_client=_UpperCamelCase			)
											mocked_bulk.return_value([(True, None)] * 3			)
											index.add_documents(["""foo""", """bar""", """foobar"""]			)

											# single query
											snake_case_		:							Union[str, Any]			       =						"""foo"""
											snake_case_		:							Any			       =						{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
											snake_case_		:							List[str]			       =						index.search(_UpperCamelCase			)
											self.assertEqual(scores[0]  ,1			)
											self.assertEqual(indices[0]  ,0			)

											# single query with timeout
											snake_case_		:							str			       =						"""foo"""
											snake_case_		:							List[Any]			       =						{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
											snake_case_		:							int			       =						index.search(_UpperCamelCase  ,request_timeout=3_0			)
											self.assertEqual(scores[0]  ,1			)
											self.assertEqual(indices[0]  ,0			)

											# batched queries
											snake_case_		:							Any			       =						["""foo""", """bar""", """foobar"""]
											snake_case_		:							int			       =						{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
											snake_case_		:							Optional[Any]			       =						index.search_batch(_UpperCamelCase			)
											snake_case_		:							List[str]			       =						[scores[0] for scores in total_scores]
											snake_case_		:							Tuple			       =						[indices[0] for indices in total_indices]
											self.assertGreater(np.min(_UpperCamelCase			)  ,0			)
											self.assertListEqual([1, 1, 1]  ,_UpperCamelCase			)

											# batched queries with timeout
											snake_case_		:							List[Any]			       =						["""foo""", """bar""", """foobar"""]
											snake_case_		:							Tuple			       =						{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
											snake_case_		:							str			       =						index.search_batch(_UpperCamelCase  ,request_timeout=3_0			)
											snake_case_		:							int			       =						[scores[0] for scores in total_scores]
											snake_case_		:							Union[str, Any]			       =						[indices[0] for indices in total_indices]
											self.assertGreater(np.min(_UpperCamelCase			)  ,0			)
											self.assertListEqual([1, 1, 1]  ,_UpperCamelCase			)



'''simple docstring'''






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


__A	:	Any     =     logging.getLogger(__name__)
class 						__UpperCamelCase	(						lowercase__							):





							def 		a__   (  self		:int  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Optional[Any]  ,_UpperCamelCase		:List[Any]=None  ,_UpperCamelCase		:Any=None			):
									snake_case_		:							List[str]			       =						self.layer[current_layer](_UpperCamelCase  ,_UpperCamelCase  ,head_mask[current_layer]			)

									snake_case_		:							Tuple			       =						layer_outputs[0]

									return hidden_states





@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'						,  lowercase__						,  )
class 						__UpperCamelCase	(						lowercase__							):





							def __init__(  self		:Dict  ,_UpperCamelCase		:Optional[Any]			):
									super().__init__(_UpperCamelCase			)

									snake_case_		:							List[Any]			       =						BertEncoderWithPabee(_UpperCamelCase			)

									self.init_weights()
									snake_case_		:							int			       =						0
									snake_case_		:							Optional[Any]			       =						0
									snake_case_		:							Any			       =						0

									snake_case_		:							str			       =						0





							def 		a__   (  self		:Tuple  ,_UpperCamelCase		:int			):
									snake_case_		:							Optional[Any]			       =						threshold





							def 		a__   (  self		:Optional[Any]  ,_UpperCamelCase		:Union[str, Any]			):
									snake_case_		:							str			       =						patience





							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							List[str]			       =						0
									snake_case_		:							List[Any]			       =						0





							def 		a__   (  self		:List[Any]			):
									snake_case_		:							str			       =						self.inference_layers_num / self.inference_instances_num
									snake_case_		:							List[str]			       =						(
									    F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
									    F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
									)
									print(_UpperCamelCase			)





							@add_start_docstrings_to_model_forward(_UpperCamelCase			)
							def 		a__   (  self		:Optional[Any]  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:List[str]=None  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:int=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:int=None  ,_UpperCamelCase		:Union[str, Any]=None  ,_UpperCamelCase		:int=False  ,):

									if input_ids is not None and inputs_embeds is not None:
											raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""			)
									elif input_ids is not None:
											snake_case_		:							Any			       =						input_ids.size()
									elif inputs_embeds is not None:
											snake_case_		:							Union[str, Any]			       =						inputs_embeds.size()[:-1]
									else:
											raise ValueError("""You have to specify either input_ids or inputs_embeds"""			)

									snake_case_		:							Tuple			       =						input_ids.device if input_ids is not None else inputs_embeds.device

									if attention_mask is None:
											snake_case_		:							Any			       =						torch.ones(_UpperCamelCase  ,device=_UpperCamelCase			)
									if token_type_ids is None:
											snake_case_		:							str			       =						torch.zeros(_UpperCamelCase  ,dtype=torch.long  ,device=_UpperCamelCase			)

									# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
									# ourselves in which case we just need to make it broadcastable to all heads.
									snake_case_		:							torch.Tensor			       =						self.get_extended_attention_mask(_UpperCamelCase  ,_UpperCamelCase  ,_UpperCamelCase			)

									# If a 2D ou 3D attention mask is provided for the cross-attention
									# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
									if self.config.is_decoder and encoder_hidden_states is not None:
											snake_case_       ,      snake_case_       ,      snake_case_		:							List[Any]			       =						encoder_hidden_states.size()
											snake_case_		:							str			       =						(encoder_batch_size, encoder_sequence_length)
											if encoder_attention_mask is None:
													snake_case_		:							List[str]			       =						torch.ones(_UpperCamelCase  ,device=_UpperCamelCase			)
											snake_case_		:							Optional[Any]			       =						self.invert_attention_mask(_UpperCamelCase			)
									else:
											snake_case_		:							List[str]			       =						None

									# Prepare head mask if needed
									# 1.0 in head_mask indicate we keep the head
									# attention_probs has shape bsz x n_heads x N x N
									# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
									# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
									snake_case_		:							List[Any]			       =						self.get_head_mask(_UpperCamelCase  ,self.config.num_hidden_layers			)

									snake_case_		:							Dict			       =						self.embeddings(
									    input_ids=_UpperCamelCase  ,position_ids=_UpperCamelCase  ,token_type_ids=_UpperCamelCase  ,inputs_embeds=_UpperCamelCase			)
									snake_case_		:							Optional[int]			       =						embedding_output

									if self.training:
											snake_case_		:							Any			       =						[]
											for i in range(self.config.num_hidden_layers			):
													snake_case_		:							Any			       =						self.encoder.adaptive_forward(
													    _UpperCamelCase  ,current_layer=_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,head_mask=_UpperCamelCase			)

													snake_case_		:							Dict			       =						self.pooler(_UpperCamelCase			)
													snake_case_		:							str			       =						output_layers[i](output_dropout(_UpperCamelCase			)			)
													res.append(_UpperCamelCase			)
									elif self.patience == 0:  # Use all layers for inference
											snake_case_		:							List[str]			       =						self.encoder(
											    _UpperCamelCase  ,attention_mask=_UpperCamelCase  ,head_mask=_UpperCamelCase  ,encoder_hidden_states=_UpperCamelCase  ,encoder_attention_mask=_UpperCamelCase  ,)
											snake_case_		:							int			       =						self.pooler(encoder_outputs[0]			)
											snake_case_		:							Tuple			       =						[output_layers[self.config.num_hidden_layers - 1](_UpperCamelCase			)]
									else:
											snake_case_		:							Any			       =						0
											snake_case_		:							List[str]			       =						None
											snake_case_		:							Any			       =						0
											for i in range(self.config.num_hidden_layers			):
													calculated_layer_num += 1
													snake_case_		:							Dict			       =						self.encoder.adaptive_forward(
													    _UpperCamelCase  ,current_layer=_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,head_mask=_UpperCamelCase			)

													snake_case_		:							Union[str, Any]			       =						self.pooler(_UpperCamelCase			)
													snake_case_		:							Optional[Any]			       =						output_layers[i](_UpperCamelCase			)
													if regression:
															snake_case_		:							Union[str, Any]			       =						logits.detach()
															if patient_result is not None:
																	snake_case_		:							Union[str, Any]			       =						patient_result.detach()
															if (patient_result is not None) and torch.abs(patient_result - labels			) < self.regression_threshold:
																	patient_counter += 1
															else:
																	snake_case_		:							Union[str, Any]			       =						0
													else:
															snake_case_		:							Any			       =						logits.detach().argmax(dim=1			)
															if patient_result is not None:
																	snake_case_		:							str			       =						patient_result.detach().argmax(dim=1			)
															if (patient_result is not None) and torch.all(labels.eq(_UpperCamelCase			)			):
																	patient_counter += 1
															else:
																	snake_case_		:							List[str]			       =						0

													snake_case_		:							List[Any]			       =						logits
													if patient_counter == self.patience:
															break
											snake_case_		:							int			       =						[patient_result]
											self.inference_layers_num += calculated_layer_num
											self.inference_instances_num += 1

									return res





@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '						,  lowercase__						,  )
class 						__UpperCamelCase	(						lowercase__							):





							def __init__(  self		:Tuple  ,_UpperCamelCase		:List[str]			):
									super().__init__(_UpperCamelCase			)
									snake_case_		:							List[str]			       =						config.num_labels

									snake_case_		:							Dict			       =						BertModelWithPabee(_UpperCamelCase			)
									snake_case_		:							List[str]			       =						nn.Dropout(config.hidden_dropout_prob			)
									snake_case_		:							str			       =						nn.ModuleList(
									    [nn.Linear(config.hidden_size  ,self.config.num_labels			) for _ in range(config.num_hidden_layers			)]			)

									self.init_weights()





							@add_start_docstrings_to_model_forward(_UpperCamelCase			)
							def 		a__   (  self		:Union[str, Any]  ,_UpperCamelCase		:Tuple=None  ,_UpperCamelCase		:Dict=None  ,_UpperCamelCase		:Union[str, Any]=None  ,_UpperCamelCase		:Union[str, Any]=None  ,_UpperCamelCase		:Tuple=None  ,_UpperCamelCase		:int=None  ,_UpperCamelCase		:Union[str, Any]=None  ,):
									snake_case_		:							int			       =						self.bert(
									    input_ids=_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,token_type_ids=_UpperCamelCase  ,position_ids=_UpperCamelCase  ,head_mask=_UpperCamelCase  ,inputs_embeds=_UpperCamelCase  ,output_dropout=self.dropout  ,output_layers=self.classifiers  ,regression=self.num_labels == 1  ,)

									snake_case_		:							Optional[int]			       =						(logits[-1],)

									if labels is not None:
											snake_case_		:							Optional[Any]			       =						None
											snake_case_		:							Optional[int]			       =						0
											for ix, logits_item in enumerate(_UpperCamelCase			):
													if self.num_labels == 1:
															#  We are doing regression
															snake_case_		:							Dict			       =						MSELoss()
															snake_case_		:							List[str]			       =						loss_fct(logits_item.view(-1			)  ,labels.view(-1			)			)
													else:
															snake_case_		:							Any			       =						CrossEntropyLoss()
															snake_case_		:							str			       =						loss_fct(logits_item.view(-1  ,self.num_labels			)  ,labels.view(-1			)			)
													if total_loss is None:
															snake_case_		:							str			       =						loss
													else:
															total_loss += loss * (ix + 1)
													total_weights += ix + 1
											snake_case_		:							str			       =						(total_loss / total_weights,) + outputs

									return outputs