


from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__     					=			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     					=			{
    "facebook/s2t-small-librispeech-asr": (
        "https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/config.json"
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech_to_text
}



class 						_UpperCAmelCase      (     A__							):
		lowerCamelCase_	:			List[Any]       			=  'speech_to_text'
		lowerCamelCase_	:			Optional[Any]       			=  ['past_key_values']
		lowerCamelCase_	:			int       			=  {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}






		def __init__(						self				:     List[Any]   ,  UpperCAmelCase				:     Union[str, Any]=1_00_00   ,  UpperCAmelCase				:     List[Any]=12   ,  UpperCAmelCase				:     List[Any]=20_48   ,  UpperCAmelCase				:     Optional[int]=4   ,  UpperCAmelCase				:     Dict=6   ,  UpperCAmelCase				:     Any=20_48   ,  UpperCAmelCase				:     Union[str, Any]=4   ,  UpperCAmelCase				:     Dict=0.0   ,  UpperCAmelCase				:     str=0.0   ,  UpperCAmelCase				:     Optional[Any]=True   ,  UpperCAmelCase				:     Any=True   ,  UpperCAmelCase				:     Union[str, Any]="relu"   ,  UpperCAmelCase				:     str=2_56   ,  UpperCAmelCase				:     Any=0.1   ,  UpperCAmelCase				:     Tuple=0.0   ,  UpperCAmelCase				:     Dict=0.0   ,  UpperCAmelCase				:     int=0.02   ,  UpperCAmelCase				:     Dict=2   ,  UpperCAmelCase				:     Dict=True   ,  UpperCAmelCase				:     Any=1   ,  UpperCAmelCase				:     Dict=0   ,  UpperCAmelCase				:     List[Any]=2   ,  UpperCAmelCase				:     List[Any]=60_00   ,  UpperCAmelCase				:     List[str]=10_24   ,  UpperCAmelCase				:     List[Any]=2   ,  UpperCAmelCase				:     Optional[int]=(5, 5)   ,  UpperCAmelCase				:     List[str]=10_24   ,  UpperCAmelCase				:     List[str]=80   ,  UpperCAmelCase				:     Optional[int]=1   ,  **UpperCAmelCase				:     Optional[Any]   ,  ):
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						vocab_size
				SCREAMING_SNAKE_CASE_				:Tuple				     =						d_model
				SCREAMING_SNAKE_CASE_				:List[str]				     =						encoder_ffn_dim
				SCREAMING_SNAKE_CASE_				:str				     =						encoder_layers
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						encoder_attention_heads
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						decoder_ffn_dim
				SCREAMING_SNAKE_CASE_				:int				     =						decoder_layers
				SCREAMING_SNAKE_CASE_				:str				     =						decoder_attention_heads
				SCREAMING_SNAKE_CASE_				:Dict				     =						dropout
				SCREAMING_SNAKE_CASE_				:int				     =						attention_dropout
				SCREAMING_SNAKE_CASE_				:Tuple				     =						activation_dropout
				SCREAMING_SNAKE_CASE_				:int				     =						activation_function
				SCREAMING_SNAKE_CASE_				:str				     =						init_std
				SCREAMING_SNAKE_CASE_				:int				     =						encoder_layerdrop
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						decoder_layerdrop
				SCREAMING_SNAKE_CASE_				:Dict				     =						use_cache
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						encoder_layers
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						scale_embedding  # scale factor will be sqrt(d_model) if True
				SCREAMING_SNAKE_CASE_				:Tuple				     =						max_source_positions
				SCREAMING_SNAKE_CASE_				:List[str]				     =						max_target_positions
				SCREAMING_SNAKE_CASE_				:Any				     =						num_conv_layers
				SCREAMING_SNAKE_CASE_				:Any				     =						list(__UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Tuple				     =						conv_channels
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						input_feat_per_channel
				SCREAMING_SNAKE_CASE_				:Any				     =						input_channels

				if len(self.conv_kernel_sizes) != self.num_conv_layers:
						raise ValueError(
						    "Configuration for convolutional module is incorrect. "
						    "It is required that `len(config.conv_kernel_sizes)` == `config.num_conv_layers` "
						    F"but is `len(config.conv_kernel_sizes) = {len(self.conv_kernel_sizes)}`, "
						    F"`config.num_conv_layers = {self.num_conv_layers}`.")

				super().__init__(
				    pad_token_id=__UpperCAmelCase   ,  bos_token_id=__UpperCAmelCase   ,  eos_token_id=__UpperCAmelCase   ,  is_encoder_decoder=__UpperCAmelCase   ,  decoder_start_token_id=__UpperCAmelCase   ,  **__UpperCAmelCase   ,  )









"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


A_				:      List[str]           =			pytest.mark.integration





@require_faiss
class  lowerCamelCase    (A__      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(__UpperCAmelCase							) for x in np.arange(3_0							).tolist()]}							)
						return dset

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Tuple:
						import faiss

						SCREAMING_SNAKE_CASE__										=   self._create_dummy_dataset()
						SCREAMING_SNAKE_CASE__										=   dset.map(
						    lambda __UpperCAmelCase     ,					__UpperCAmelCase							: {"vecs": i * np.ones(5     ,					dtype=np.floataa							)}     ,					with_indices=__UpperCAmelCase     ,					keep_in_memory=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   dset.add_faiss_index("""vecs"""     ,					batch_size=1_0_0     ,					metric_type=faiss.METRIC_INNER_PRODUCT							)
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   dset.get_nearest_examples("""vecs"""     ,					np.ones(5     ,					dtype=np.floataa							)							)
						self.assertEqual(examples["""filename"""][0]     ,					"""my_name-train_29"""							)
						dset.drop_index("""vecs"""							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     str:
						import faiss

						SCREAMING_SNAKE_CASE__										=   self._create_dummy_dataset()
						dset.add_faiss_index_from_external_arrays(
						    external_arrays=np.ones((3_0, 5)							) * np.arange(3_0							).reshape(-1     ,					1							)     ,					index_name="""vecs"""     ,					batch_size=1_0_0     ,					metric_type=faiss.METRIC_INNER_PRODUCT     ,					)
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   dset.get_nearest_examples("""vecs"""     ,					np.ones(5     ,					dtype=np.floataa							)							)
						self.assertEqual(examples["""filename"""][0]     ,					"""my_name-train_29"""							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Any:
						import faiss

						SCREAMING_SNAKE_CASE__										=   self._create_dummy_dataset()
						dset.add_faiss_index_from_external_arrays(
						    external_arrays=np.ones((3_0, 5)							) * np.arange(3_0							).reshape(-1     ,					1							)     ,					index_name="""vecs"""     ,					metric_type=faiss.METRIC_INNER_PRODUCT     ,					)

						# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
						# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
						# see https://bugs.python.org/issue14243 and
						# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
						with tempfile.NamedTemporaryFile(delete=__UpperCAmelCase							) as tmp_file:
									dset.save_faiss_index("""vecs"""     ,					tmp_file.name							)
									dset.load_faiss_index("""vecs2"""     ,					tmp_file.name							)
						os.unlink(tmp_file.name							)

						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   dset.get_nearest_examples("""vecs2"""     ,					np.ones(5     ,					dtype=np.floataa							)							)
						self.assertEqual(examples["""filename"""][0]     ,					"""my_name-train_29"""							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   self._create_dummy_dataset()
						dset.add_faiss_index_from_external_arrays(
						    external_arrays=np.ones((3_0, 5)							) * np.arange(3_0							).reshape(-1     ,					1							)     ,					index_name="""vecs"""							)
						dset.drop_index("""vecs"""							)
						self.assertRaises(__UpperCAmelCase     ,					partial(dset.get_nearest_examples     ,					"""vecs2"""     ,					np.ones(5     ,					dtype=np.floataa							)							)							)





			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     str:
						from elasticsearch import Elasticsearch

						SCREAMING_SNAKE_CASE__										=   self._create_dummy_dataset()
						with patch("""elasticsearch.Elasticsearch.search"""							) as mocked_search, patch(
						    """elasticsearch.client.IndicesClient.create"""							) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""							) as mocked_bulk:
									SCREAMING_SNAKE_CASE__										=   {"""acknowledged""": True}
									mocked_bulk.return_value([(True, None)] * 3_0							)
									SCREAMING_SNAKE_CASE__										=   {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 2_9}]}}
									SCREAMING_SNAKE_CASE__										=   Elasticsearch()

									dset.add_elasticsearch_index("""filename"""     ,					es_client=__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   dset.get_nearest_examples("""filename"""     ,					"""my_name-train_29"""							)
									self.assertEqual(examples["""filename"""][0]     ,					"""my_name-train_29"""							)





@require_faiss
class  lowerCamelCase    (A__      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						import faiss

						SCREAMING_SNAKE_CASE__										=   FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT							)

						# add vectors
						index.add_vectors(np.eye(5     ,					dtype=np.floataa							)							)
						self.assertIsNotNone(index.faiss_index							)
						self.assertEqual(index.faiss_index.ntotal     ,					5							)
						index.add_vectors(np.zeros((5, 5)     ,					dtype=np.floataa							)							)
						self.assertEqual(index.faiss_index.ntotal     ,					1_0							)

						# single query
						SCREAMING_SNAKE_CASE__										=   np.zeros(5     ,					dtype=np.floataa							)
						SCREAMING_SNAKE_CASE__										=   1
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search(__UpperCAmelCase							)
						self.assertRaises(__UpperCAmelCase     ,					index.search     ,					query.reshape(-1     ,					1							)							)
						self.assertGreater(scores[0]     ,					0							)
						self.assertEqual(indices[0]     ,					1							)

						# batched queries
						SCREAMING_SNAKE_CASE__										=   np.eye(5     ,					dtype=np.floataa							)[::-1]
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search_batch(__UpperCAmelCase							)
						self.assertRaises(__UpperCAmelCase     ,					index.search_batch     ,					queries[0]							)
						SCREAMING_SNAKE_CASE__										=   [scores[0] for scores in total_scores]
						SCREAMING_SNAKE_CASE__										=   [indices[0] for indices in total_indices]
						self.assertGreater(np.min(__UpperCAmelCase							)     ,					0							)
						self.assertListEqual([4, 3, 2, 1, 0]     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     str:
						import faiss

						SCREAMING_SNAKE_CASE__										=   FaissIndex(string_factory="""Flat"""							)
						index.add_vectors(np.eye(5     ,					dtype=np.floataa							)							)
						self.assertIsInstance(index.faiss_index     ,					faiss.IndexFlat							)
						SCREAMING_SNAKE_CASE__										=   FaissIndex(string_factory="""LSH"""							)
						index.add_vectors(np.eye(5     ,					dtype=np.floataa							)							)
						self.assertIsInstance(index.faiss_index     ,					faiss.IndexLSH							)
						with self.assertRaises(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   FaissIndex(string_factory="""Flat"""     ,					custom_index=faiss.IndexFlat(5							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Any:
						import faiss

						SCREAMING_SNAKE_CASE__										=   faiss.IndexFlat(5							)
						SCREAMING_SNAKE_CASE__										=   FaissIndex(custom_index=__UpperCAmelCase							)
						index.add_vectors(np.eye(5     ,					dtype=np.floataa							)							)
						self.assertIsInstance(index.faiss_index     ,					faiss.IndexFlat							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     str:
						import faiss

						SCREAMING_SNAKE_CASE__										=   FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT							)
						index.add_vectors(np.eye(5     ,					dtype=np.floataa							)							)

						# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
						# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
						# see https://bugs.python.org/issue14243 and
						# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
						with tempfile.NamedTemporaryFile(delete=__UpperCAmelCase							) as tmp_file:
									index.save(tmp_file.name							)
									SCREAMING_SNAKE_CASE__										=   FaissIndex.load(tmp_file.name							)
						os.unlink(tmp_file.name							)

						SCREAMING_SNAKE_CASE__										=   np.zeros(5     ,					dtype=np.floataa							)
						SCREAMING_SNAKE_CASE__										=   1
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search(__UpperCAmelCase							)
						self.assertGreater(scores[0]     ,					0							)
						self.assertEqual(indices[0]     ,					1							)

@require_faiss
def 						A							(  snake_case__     ):




			'''simple docstring'''


			import faiss

			SCREAMING_SNAKE_CASE__										=   FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
			index.add_vectors(np.eye(5		,					dtype=np.floataa     )     )

			SCREAMING_SNAKE_CASE__										=   """index.faiss"""
			SCREAMING_SNAKE_CASE__										=   f"""mock://{index_name}"""
			index.save(snake_case__		,					storage_options=mockfs.storage_options     )
			SCREAMING_SNAKE_CASE__										=   FaissIndex.load(snake_case__		,					storage_options=mockfs.storage_options     )

			SCREAMING_SNAKE_CASE__										=   np.zeros(5		,					dtype=np.floataa     )
			SCREAMING_SNAKE_CASE__										=   1
			SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search(snake_case__     )
			assert scores[0] > 0
			assert indices[0] == 1





@require_elasticsearch
class  lowerCamelCase    (A__      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     Dict:
						from elasticsearch import Elasticsearch

						with patch("""elasticsearch.Elasticsearch.search"""							) as mocked_search, patch(
						    """elasticsearch.client.IndicesClient.create"""							) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""							) as mocked_bulk:
									SCREAMING_SNAKE_CASE__										=   Elasticsearch()
									SCREAMING_SNAKE_CASE__										=   {"""acknowledged""": True}
									SCREAMING_SNAKE_CASE__										=   ElasticSearchIndex(es_client=__UpperCAmelCase							)
									mocked_bulk.return_value([(True, None)] * 3							)
									index.add_documents(["""foo""", """bar""", """foobar"""]							)

									# single query
									SCREAMING_SNAKE_CASE__										=   """foo"""
									SCREAMING_SNAKE_CASE__										=   {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
									SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search(__UpperCAmelCase							)
									self.assertEqual(scores[0]     ,					1							)
									self.assertEqual(indices[0]     ,					0							)

									# single query with timeout
									SCREAMING_SNAKE_CASE__										=   """foo"""
									SCREAMING_SNAKE_CASE__										=   {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
									SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search(__UpperCAmelCase     ,					request_timeout=3_0							)
									self.assertEqual(scores[0]     ,					1							)
									self.assertEqual(indices[0]     ,					0							)

									# batched queries
									SCREAMING_SNAKE_CASE__										=   ["""foo""", """bar""", """foobar"""]
									SCREAMING_SNAKE_CASE__										=   {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
									SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search_batch(__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__										=   [scores[0] for scores in total_scores]
									SCREAMING_SNAKE_CASE__										=   [indices[0] for indices in total_indices]
									self.assertGreater(np.min(__UpperCAmelCase							)     ,					0							)
									self.assertListEqual([1, 1, 1]     ,					__UpperCAmelCase							)

									# batched queries with timeout
									SCREAMING_SNAKE_CASE__										=   ["""foo""", """bar""", """foobar"""]
									SCREAMING_SNAKE_CASE__										=   {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
									SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   index.search_batch(__UpperCAmelCase     ,					request_timeout=3_0							)
									SCREAMING_SNAKE_CASE__										=   [scores[0] for scores in total_scores]
									SCREAMING_SNAKE_CASE__										=   [indices[0] for indices in total_indices]
									self.assertGreater(np.min(__UpperCAmelCase							)     ,					0							)
									self.assertListEqual([1, 1, 1]     ,					__UpperCAmelCase							)


