

'''simple docstring'''





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class 				lowercase       ( UpperCAmelCase__    ):




				def 					a							(      self ):
											snake_case_           =					tempfile.mkdtemp()
											snake_case_           =					5

											# Realm tok
											snake_case_           =					[
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'test',
											    'question',
											    'this',
											    'is',
											    'the',
											    'first',
											    'second',
											    'third',
											    'fourth',
											    'fifth',
											    'record',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case_           =					os.path.join(self.tmpdirname   ,     'realm_tokenizer' )
											os.makedirs(__lowerCAmelCase   ,     exist_ok=__lowerCAmelCase )
											snake_case_           =					os.path.join(__lowerCAmelCase   ,     VOCAB_FILES_NAMES['vocab_file'] )
											with open(self.vocab_file   ,     'w'   ,     encoding='utf-8' ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )

											snake_case_           =					os.path.join(self.tmpdirname   ,     'realm_block_records' )
											os.makedirs(__lowerCAmelCase   ,     exist_ok=__lowerCAmelCase )




				def 					a							(      self ):
											return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,     'realm_tokenizer' ) )




				def 					a							(      self ):
											shutil.rmtree(self.tmpdirname )




				def 					a							(      self ):
											snake_case_           =					RealmConfig(num_block_records=self.num_block_records )
											return config




				def 					a							(      self ):
											snake_case_           =					Dataset.from_dict(
											    {
											        'id': ['0', '1'],
											        'question': ['foo', 'bar'],
											        'answers': [['Foo', 'Bar'], ['Bar']],
											    } )
											return dataset




				def 					a							(      self ):
											snake_case_           =					np.array(
											    [
											        b'This is the first record',
											        b'This is the second record',
											        b'This is the third record',
											        b'This is the fourth record',
											        b'This is the fifth record',
											        b'This is a longer longer longer record',
											    ]   ,     dtype=__lowerCAmelCase   ,     )
											return block_records




				def 					a							(      self ):
											snake_case_           =					RealmRetriever(
											    block_records=self.get_dummy_block_records()   ,     tokenizer=self.get_tokenizer()   ,     )
											return retriever




				def 					a							(      self ):
											snake_case_           =					self.get_config()
											snake_case_           =					self.get_dummy_retriever()
											snake_case_           =					retriever.tokenizer

											snake_case_           =					np.array([0, 3]   ,     dtype='long' )
											snake_case_           =					tokenizer(['Test question'] ).input_ids
											snake_case_           =					tokenizer(
											    ['the fourth']   ,     add_special_tokens=__lowerCAmelCase   ,     return_token_type_ids=__lowerCAmelCase   ,     return_attention_mask=__lowerCAmelCase   ,     ).input_ids
											snake_case_           =					config.reader_seq_len

											snake_case_       ,					snake_case_       ,					snake_case_       ,					snake_case_           =					retriever(
											    __lowerCAmelCase   ,     __lowerCAmelCase   ,     answer_ids=__lowerCAmelCase   ,     max_length=__lowerCAmelCase   ,     return_tensors='np' )

											self.assertEqual(len(__lowerCAmelCase )   ,     2 )
											self.assertEqual(len(__lowerCAmelCase )   ,     2 )
											self.assertEqual(len(__lowerCAmelCase )   ,     2 )
											self.assertEqual(concat_inputs.input_ids.shape   ,     (2, 10) )
											self.assertEqual(concat_inputs.attention_mask.shape   ,     (2, 10) )
											self.assertEqual(concat_inputs.token_type_ids.shape   ,     (2, 10) )
											self.assertEqual(concat_inputs.special_tokens_mask.shape   ,     (2, 10) )
											self.assertEqual(
											    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0] )   ,     ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']   ,     )
											self.assertEqual(
											    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1] )   ,     ['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']   ,     )




				def 					a							(      self ):
											snake_case_           =					self.get_config()
											snake_case_           =					self.get_dummy_retriever()
											snake_case_           =					retriever.tokenizer

											snake_case_           =					np.array([0, 3, 5]   ,     dtype='long' )
											snake_case_           =					tokenizer(['Test question'] ).input_ids
											snake_case_           =					tokenizer(
											    ['the fourth', 'longer longer']   ,     add_special_tokens=__lowerCAmelCase   ,     return_token_type_ids=__lowerCAmelCase   ,     return_attention_mask=__lowerCAmelCase   ,     ).input_ids
											snake_case_           =					config.reader_seq_len

											snake_case_       ,					snake_case_       ,					snake_case_       ,					snake_case_           =					retriever(
											    __lowerCAmelCase   ,     __lowerCAmelCase   ,     answer_ids=__lowerCAmelCase   ,     max_length=__lowerCAmelCase   ,     return_tensors='np' )

											self.assertEqual([False, True, True]   ,     __lowerCAmelCase )
											self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]   ,     __lowerCAmelCase )
											self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]   ,     __lowerCAmelCase )




				def 					a							(      self ):
											snake_case_           =					self.get_dummy_retriever()
											retriever.save_pretrained(os.path.join(self.tmpdirname   ,     'realm_block_records' ) )

											# Test local path
											snake_case_           =					retriever.from_pretrained(os.path.join(self.tmpdirname   ,     'realm_block_records' ) )
											self.assertEqual(retriever.block_records[0]   ,     b'This is the first record' )

											# Test mocked remote path
											with patch('transformers.models.realm.retrieval_realm.hf_hub_download' ) as mock_hf_hub_download:
																		snake_case_           =					os.path.join(
																		    os.path.join(self.tmpdirname   ,     'realm_block_records' )   ,     _REALM_BLOCK_RECORDS_FILENAME )
																		snake_case_           =					RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa' )

											self.assertEqual(retriever.block_records[0]   ,     b'This is the first record' )







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_torch_available


_UpperCAmelCase	:     Any		       = {
    """configuration_longt5""": ["""LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LongT5Config""", """LongT5OnnxConfig"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase	:     Any		       = [
					    """LONGT5_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """LongT5EncoderModel""",
					    """LongT5ForConditionalGeneration""",
					    """LongT5Model""",
					    """LongT5PreTrainedModel""",
					]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase	:     Optional[int]		       = [
					    """FlaxLongT5ForConditionalGeneration""",
					    """FlaxLongT5Model""",
					    """FlaxLongT5PreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_longta import LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP, LongTaConfig, LongTaOnnxConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_longta import (
										    LONGT5_PRETRAINED_MODEL_ARCHIVE_LIST,
										    LongTaEncoderModel,
										    LongTaForConditionalGeneration,
										    LongTaModel,
										    LongTaPreTrainedModel,
										)

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_longta import (
										    FlaxLongTaForConditionalGeneration,
										    FlaxLongTaModel,
										    FlaxLongTaPreTrainedModel,
										)


else:
					import sys

					_UpperCAmelCase	:     Optional[Any]		       = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




