

import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer





class     snake_case_			(				_a							):


							"""simple docstring"""





							def        A__  ( self							):
												__lowerCAmelCase					=							tempfile.mkdtemp()
												__lowerCAmelCase					=							5

												# Realm tok
												__lowerCAmelCase					=							[
												    '[UNK]',
												    '[CLS]',
												    '[SEP]',
												    '[PAD]',
												    '[MASK]',
												    'test',
												    'question',
												    'this',
												    'is',
												    'the',
												    'first',
												    'second',
												    'third',
												    'fourth',
												    'fifth',
												    'record',
												    'want',
												    '##want',
												    '##ed',
												    'wa',
												    'un',
												    'runn',
												    '##ing',
												    ',',
												    'low',
												    'lowest',
												]
												__lowerCAmelCase					=							os.path.join(self.tmpdirname					,						'realm_tokenizer'							)
												os.makedirs(_A					,						exist_ok=_A							)
												__lowerCAmelCase					=							os.path.join(_A					,						VOCAB_FILES_NAMES['vocab_file']							)
												with open(self.vocab_file					,						'w'					,						encoding='utf-8'							) as vocab_writer:
																	vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)

												__lowerCAmelCase					=							os.path.join(self.tmpdirname					,						'realm_block_records'							)
												os.makedirs(_A					,						exist_ok=_A							)




							def        A__  ( self							):
												return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,						'realm_tokenizer'							)							)




							def        A__  ( self							):
												shutil.rmtree(self.tmpdirname							)




							def        A__  ( self							):
												__lowerCAmelCase					=							RealmConfig(num_block_records=self.num_block_records							)
												return config




							def        A__  ( self							):
												__lowerCAmelCase					=							Dataset.from_dict(
												    {
												        'id': ['0', '1'],
												        'question': ['foo', 'bar'],
												        'answers': [['Foo', 'Bar'], ['Bar']],
												    }							)
												return dataset




							def        A__  ( self							):
												__lowerCAmelCase					=							np.array(
												    [
												        b'This is the first record',
												        b'This is the second record',
												        b'This is the third record',
												        b'This is the fourth record',
												        b'This is the fifth record',
												        b'This is a longer longer longer record',
												    ]					,						dtype=_A					,						)
												return block_records




							def        A__  ( self							):
												__lowerCAmelCase					=							RealmRetriever(
												    block_records=self.get_dummy_block_records()					,						tokenizer=self.get_tokenizer()					,						)
												return retriever




							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_config()
												__lowerCAmelCase					=							self.get_dummy_retriever()
												__lowerCAmelCase					=							retriever.tokenizer

												__lowerCAmelCase					=							np.array([0, 3]					,						dtype='long'							)
												__lowerCAmelCase					=							tokenizer(['Test question']							).input_ids
												__lowerCAmelCase					=							tokenizer(
												    ['the fourth']					,						add_special_tokens=_A					,						return_token_type_ids=_A					,						return_attention_mask=_A					,						).input_ids
												__lowerCAmelCase					=							config.reader_seq_len

												__lowerCAmelCase,	__lowerCAmelCase,	__lowerCAmelCase,	__lowerCAmelCase					=							retriever(
												    _A					,						_A					,						answer_ids=_A					,						max_length=_A					,						return_tensors='np'							)

												self.assertEqual(len(_A							)					,						2							)
												self.assertEqual(len(_A							)					,						2							)
												self.assertEqual(len(_A							)					,						2							)
												self.assertEqual(concat_inputs.input_ids.shape					,						(2, 1_0)							)
												self.assertEqual(concat_inputs.attention_mask.shape					,						(2, 1_0)							)
												self.assertEqual(concat_inputs.token_type_ids.shape					,						(2, 1_0)							)
												self.assertEqual(concat_inputs.special_tokens_mask.shape					,						(2, 1_0)							)
												self.assertEqual(
												    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]							)					,						['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']					,						)
												self.assertEqual(
												    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]							)					,						['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']					,						)




							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_config()
												__lowerCAmelCase					=							self.get_dummy_retriever()
												__lowerCAmelCase					=							retriever.tokenizer

												__lowerCAmelCase					=							np.array([0, 3, 5]					,						dtype='long'							)
												__lowerCAmelCase					=							tokenizer(['Test question']							).input_ids
												__lowerCAmelCase					=							tokenizer(
												    ['the fourth', 'longer longer']					,						add_special_tokens=_A					,						return_token_type_ids=_A					,						return_attention_mask=_A					,						).input_ids
												__lowerCAmelCase					=							config.reader_seq_len

												__lowerCAmelCase,	__lowerCAmelCase,	__lowerCAmelCase,	__lowerCAmelCase					=							retriever(
												    _A					,						_A					,						answer_ids=_A					,						max_length=_A					,						return_tensors='np'							)

												self.assertEqual([False, True, True]					,						_A							)
												self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,						_A							)
												self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,						_A							)




							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_dummy_retriever()
												retriever.save_pretrained(os.path.join(self.tmpdirname					,						'realm_block_records'							)							)

												# Test local path
												__lowerCAmelCase					=							retriever.from_pretrained(os.path.join(self.tmpdirname					,						'realm_block_records'							)							)
												self.assertEqual(retriever.block_records[0]					,						b'This is the first record'							)

												# Test mocked remote path
												with patch('transformers.models.realm.retrieval_realm.hf_hub_download'							) as mock_hf_hub_download:
																	__lowerCAmelCase					=							os.path.join(
																	    os.path.join(self.tmpdirname					,						'realm_block_records'							)					,						_REALM_BLOCK_RECORDS_FILENAME							)
																	__lowerCAmelCase					=							RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'							)

												self.assertEqual(retriever.block_records[0]					,						b'This is the first record'							)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCamelCase      				=     {
    '''configuration_data2vec_audio''': ['''DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''Data2VecAudioConfig'''],
    '''configuration_data2vec_text''': [
        '''DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''Data2VecTextConfig''',
        '''Data2VecTextOnnxConfig''',
    ],
    '''configuration_data2vec_vision''': [
        '''DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''Data2VecVisionConfig''',
        '''Data2VecVisionOnnxConfig''',
    ],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase      				=     [
		    '''DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''Data2VecAudioForAudioFrameClassification''',
		    '''Data2VecAudioForCTC''',
		    '''Data2VecAudioForSequenceClassification''',
		    '''Data2VecAudioForXVector''',
		    '''Data2VecAudioModel''',
		    '''Data2VecAudioPreTrainedModel''',
		]
		lowerCamelCase      				=     [
		    '''DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''Data2VecTextForCausalLM''',
		    '''Data2VecTextForMaskedLM''',
		    '''Data2VecTextForMultipleChoice''',
		    '''Data2VecTextForQuestionAnswering''',
		    '''Data2VecTextForSequenceClassification''',
		    '''Data2VecTextForTokenClassification''',
		    '''Data2VecTextModel''',
		    '''Data2VecTextPreTrainedModel''',
		]
		lowerCamelCase      				=     [
		    '''DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''Data2VecVisionForImageClassification''',
		    '''Data2VecVisionForMaskedImageModeling''',
		    '''Data2VecVisionForSemanticSegmentation''',
		    '''Data2VecVisionModel''',
		    '''Data2VecVisionPreTrainedModel''',
		]

if is_tf_available():
		lowerCamelCase      				=     [
		    '''TFData2VecVisionForImageClassification''',
		    '''TFData2VecVisionForSemanticSegmentation''',
		    '''TFData2VecVisionModel''',
		    '''TFData2VecVisionPreTrainedModel''',
		]

if TYPE_CHECKING:
		from .configuration_dataavec_audio import DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP, DataaVecAudioConfig
		from .configuration_dataavec_text import (
		    DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    DataaVecTextConfig,
		    DataaVecTextOnnxConfig,
		)
		from .configuration_dataavec_vision import (
		    DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    DataaVecVisionConfig,
		    DataaVecVisionOnnxConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_dataavec_audio import (
				    DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST,
				    DataaVecAudioForAudioFrameClassification,
				    DataaVecAudioForCTC,
				    DataaVecAudioForSequenceClassification,
				    DataaVecAudioForXVector,
				    DataaVecAudioModel,
				    DataaVecAudioPreTrainedModel,
				)
				from .modeling_dataavec_text import (
				    DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    DataaVecTextForCausalLM,
				    DataaVecTextForMaskedLM,
				    DataaVecTextForMultipleChoice,
				    DataaVecTextForQuestionAnswering,
				    DataaVecTextForSequenceClassification,
				    DataaVecTextForTokenClassification,
				    DataaVecTextModel,
				    DataaVecTextPreTrainedModel,
				)
				from .modeling_dataavec_vision import (
				    DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST,
				    DataaVecVisionForImageClassification,
				    DataaVecVisionForMaskedImageModeling,
				    DataaVecVisionForSemanticSegmentation,
				    DataaVecVisionModel,
				    DataaVecVisionPreTrainedModel,
				)
		if is_tf_available():
				from .modeling_tf_dataavec_vision import (
				    TFDataaVecVisionForImageClassification,
				    TFDataaVecVisionForSemanticSegmentation,
				    TFDataaVecVisionModel,
				    TFDataaVecVisionPreTrainedModel,
				)

else:
		import sys

		lowerCamelCase      				=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






