




'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin





class  __lowercase    (	__SCREAMING_SNAKE_CASE							,				unittest.TestCase			):
   _a    						=   CanineTokenizer
   _a    						=   False







   def 		UpperCamelCase__      (						self   )					->     str:
     super().setUp()
     __a								=       CanineTokenizer()
     tokenizer.save_pretrained(self.tmpdirname   )







   @cached_property
   def 		UpperCamelCase__      (						self   )					->     Dict:
     return CanineTokenizer.from_pretrained('google/canine-s'   )







   def 		UpperCamelCase__      (						self   ,				**UpperCamelCase   )					->     Tuple:
     __a								=       self.tokenizer_class.from_pretrained(self.tmpdirname   ,				**lowerCAmelCase__   )
     __a								=       1024
     return tokenizer







   @require_torch
   def 		UpperCamelCase__      (						self   )					->     List[Any]:
     __a								=       self.canine_tokenizer
     __a								=       ['Life is like a box of chocolates.', 'You never know what you\'re gonna get.']
     # fmt: off
     __a								=       [5_7344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 5_7345, 0, 0, 0, 0]
     # fmt: on
     __a								=       tokenizer(lowerCAmelCase__   ,				padding=lowerCAmelCase__   ,				return_tensors='pt'   )
     self.assertIsInstance(lowerCAmelCase__   ,				lowerCAmelCase__   )

     __a								=       list(batch.input_ids.numpy()[0]   )

     self.assertListEqual(lowerCAmelCase__   ,				lowerCAmelCase__   )

     self.assertEqual((2, 39)   ,				batch.input_ids.shape   )
     self.assertEqual((2, 39)   ,				batch.attention_mask.shape   )







   @require_torch
   def 		UpperCamelCase__      (						self   )					->     Any:
     __a								=       self.canine_tokenizer
     __a								=       ['Once there was a man.', 'He wrote a test in HuggingFace Tranformers.']
     __a								=       tokenizer(lowerCAmelCase__   ,				padding=lowerCAmelCase__   ,				return_tensors='pt'   )
     # check if input_ids, attention_mask and token_type_ids are returned
     self.assertIn('input_ids'   ,				lowerCAmelCase__   )
     self.assertIn('attention_mask'   ,				lowerCAmelCase__   )
     self.assertIn('token_type_ids'   ,				lowerCAmelCase__   )







   @require_torch
   def 		UpperCamelCase__      (						self   )					->     Any:
     __a								=       self.canine_tokenizer
     __a								=       [
         'What\'s the weater?',
         'It\'s about 25 degrees.',
     ]
     __a								=       tokenizer(
         text_target=lowerCAmelCase__   ,				max_length=32   ,				padding='max_length'   ,				truncation=lowerCAmelCase__   ,				return_tensors='pt'   )
     self.assertEqual(32   ,				targets['input_ids'].shape[1]   )







   def 		UpperCamelCase__      (						self   )					->     List[str]:
     __a								=       self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         self.assertNotEqual(tokenizer.model_max_length   ,				42   )

        # Now let's start the test
     __a								=       self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         # Isolate this from the other tests because we save additional tokens/etc
         __a								=       tempfile.mkdtemp()

         __a								=       ' He is very happy, UNwant\u00E9d,running'
         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         tokenizer.save_pretrained(lowerCAmelCase__   )

         __a								=       tokenizer.__class__.from_pretrained(lowerCAmelCase__   )
         __a								=       after_tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         self.assertListEqual(lowerCAmelCase__   ,				lowerCAmelCase__   )

         shutil.rmtree(lowerCAmelCase__   )

     __a								=       self.get_tokenizers(model_max_length=42   )
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         # Isolate this from the other tests because we save additional tokens/etc
         __a								=       tempfile.mkdtemp()

         __a								=       ' He is very happy, UNwant\u00E9d,running'

         __a								=       tokenizer.additional_special_tokens

         # We can add a new special token for Canine as follows:
         __a								=       chr(0xE_0_0_7   )
         additional_special_tokens.append(lowerCAmelCase__   )
         tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}   )
         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         tokenizer.save_pretrained(lowerCAmelCase__   )

         __a								=       tokenizer.__class__.from_pretrained(lowerCAmelCase__   )
         __a								=       after_tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         self.assertListEqual(lowerCAmelCase__   ,				lowerCAmelCase__   )
         self.assertIn(lowerCAmelCase__   ,				after_tokenizer.additional_special_tokens   )
         self.assertEqual(after_tokenizer.model_max_length   ,				42   )

         __a								=       tokenizer.__class__.from_pretrained(lowerCAmelCase__   ,				model_max_length=43   )
         self.assertEqual(tokenizer.model_max_length   ,				43   )

         shutil.rmtree(lowerCAmelCase__   )







   def 		UpperCamelCase__      (						self   )					->     str:
     __a								=       self.get_tokenizers(do_lower_case=lowerCAmelCase__   )
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         __a								=       self.get_clean_sequence(lowerCAmelCase__   )

         # a special token for Canine can be defined as follows:
         __a								=       0xE_0_0_5
         __a								=       chr(lowerCAmelCase__   )

         tokenizer.add_special_tokens({'cls_token': special_token}   )
         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         self.assertEqual(len(lowerCAmelCase__   )   ,				1   )

         __a								=       tokenizer.decode(ids + encoded_special_token   ,				clean_up_tokenization_spaces=lowerCAmelCase__   )
         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )

         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         self.assertEqual(lowerCAmelCase__   ,				input_encoded + special_token_id   )

         __a								=       tokenizer.decode(lowerCAmelCase__   ,				skip_special_tokens=lowerCAmelCase__   )
         self.assertTrue(special_token not in decoded   )







   def 		UpperCamelCase__      (						self   )					->     Tuple:
     __a								=       self.get_tokenizers(do_lower_case=lowerCAmelCase__   )
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         __a								=       chr(0xE_0_0_5   )
         __a								=       chr(0xE_0_0_6   )

         # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
         tokenizer.add_tokens([SPECIAL_TOKEN_1]   ,				special_tokens=lowerCAmelCase__   )
         # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
         # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
         tokenizer.add_special_tokens({'additional_special_tokens': [SPECIAL_TOKEN_2]}   )

         __a								=       tokenizer.tokenize(lowerCAmelCase__   )
         __a								=       tokenizer.tokenize(lowerCAmelCase__   )

         self.assertEqual(len(lowerCAmelCase__   )   ,				1   )
         self.assertEqual(len(lowerCAmelCase__   )   ,				1   )
         self.assertEqual(token_a[0]   ,				lowerCAmelCase__   )
         self.assertEqual(token_a[0]   ,				lowerCAmelCase__   )







   @require_tokenizers
   def 		UpperCamelCase__      (						self   )					->     int:
     __a								=       self.get_tokenizers(do_lower_case=lowerCAmelCase__   )
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         # a special token for Canine can be defined as follows:
         __a								=       0xE_0_0_6
         __a								=       chr(lowerCAmelCase__   )

         __a								=       AddedToken(lowerCAmelCase__   ,				lstrip=lowerCAmelCase__   )
         tokenizer.add_special_tokens({'additional_special_tokens': [new_token]}   )

         with tempfile.TemporaryDirectory() as tmp_dir_name:
           tokenizer.save_pretrained(lowerCAmelCase__   )
           tokenizer.from_pretrained(lowerCAmelCase__   )







   def 		UpperCamelCase__      (						self   )					->     Optional[Any]:
     __a								=       []
     if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

     if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

     for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer_utils.save_pretrained(lowerCAmelCase__   )

         with open(os.path.join(lowerCAmelCase__   ,				'special_tokens_map.json'   )   ,				encoding='utf-8'   ) as json_file:
           __a								=       json.load(lowerCAmelCase__   )

         with open(os.path.join(lowerCAmelCase__   ,				'tokenizer_config.json'   )   ,				encoding='utf-8'   ) as json_file:
           __a								=       json.load(lowerCAmelCase__   )

         # a special token for Canine can be defined as follows:
         __a								=       0xE_0_0_6
         __a								=       chr(lowerCAmelCase__   )

         __a								=       [new_token_a]
         __a								=       [new_token_a]

         with open(os.path.join(lowerCAmelCase__   ,				'special_tokens_map.json'   )   ,				'w'   ,				encoding='utf-8'   ) as outfile:
           json.dump(lowerCAmelCase__   ,				lowerCAmelCase__   )
         with open(os.path.join(lowerCAmelCase__   ,				'tokenizer_config.json'   )   ,				'w'   ,				encoding='utf-8'   ) as outfile:
           json.dump(lowerCAmelCase__   ,				lowerCAmelCase__   )

         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
         # "special_tokens_map.json" files
         __a								=       tokenizer_class.from_pretrained(lowerCAmelCase__   ,				extra_ids=0   )
         self.assertIn(lowerCAmelCase__   ,				tokenizer_without_change_in_init.additional_special_tokens   )
         # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
         self.assertEqual(
             [new_token_a]   ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                 tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]   )   )   ,				)

         __a								=       0xE_0_0_7
         __a								=       chr(lowerCAmelCase__   )
         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
         __a								=       [AddedToken(lowerCAmelCase__   ,				lstrip=lowerCAmelCase__   )]
         __a								=       tokenizer_class.from_pretrained(
             lowerCAmelCase__   ,				additional_special_tokens=lowerCAmelCase__   ,				extra_ids=0   )

         self.assertIn(lowerCAmelCase__   ,				tokenizer.additional_special_tokens   )
         # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
         self.assertEqual(
             [new_token_a]   ,				tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]   )   )   )







   @require_tokenizers
   def 		UpperCamelCase__      (						self   )					->     int:
     __a								=       self.get_tokenizers(do_lower_case=lowerCAmelCase__   )
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         __a								=       'hello world'
         if self.space_between_special_tokens:
           __a								=       '[CLS] hello world [SEP]'
         else:
           __a								=       input
         __a								=       tokenizer.encode(lowerCAmelCase__   ,				add_special_tokens=lowerCAmelCase__   )
         __a								=       tokenizer.decode(lowerCAmelCase__   ,				spaces_between_special_tokens=self.space_between_special_tokens   )
         self.assertIn(lowerCAmelCase__   ,				[output, output.lower()]   )







   def 		UpperCamelCase__      (						self   )					->     List[Any]:
     __a								=       self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"{tokenizer.__class__.__name__}"   ):
         __a								=       [
             'bos_token',
             'eos_token',
             'unk_token',
             'sep_token',
             'pad_token',
             'cls_token',
             'mask_token',
         ]

     __a								=       'a'
     __a								=       ord(lowerCAmelCase__   )

     for attr in attributes_list:
       setattr(lowerCAmelCase__   ,				attr + '_id'   ,				lowerCAmelCase__   )
       self.assertEqual(getattr(lowerCAmelCase__   ,				lowerCAmelCase__   )   ,				lowerCAmelCase__   )
       self.assertEqual(getattr(lowerCAmelCase__   ,				attr + '_id'   )   ,				lowerCAmelCase__   )

       setattr(lowerCAmelCase__   ,				attr + '_id'   ,				lowerCAmelCase__   )
       self.assertEqual(getattr(lowerCAmelCase__   ,				lowerCAmelCase__   )   ,				lowerCAmelCase__   )
       self.assertEqual(getattr(lowerCAmelCase__   ,				attr + '_id'   )   ,				lowerCAmelCase__   )

     setattr(lowerCAmelCase__   ,				'additional_special_tokens_ids'   ,				[]   )
     self.assertListEqual(getattr(lowerCAmelCase__   ,				'additional_special_tokens'   )   ,				[]   )
     self.assertListEqual(getattr(lowerCAmelCase__   ,				'additional_special_tokens_ids'   )   ,				[]   )

     __a								=       0xE_0_0_6
     __a								=       chr(lowerCAmelCase__   )
     setattr(lowerCAmelCase__   ,				'additional_special_tokens_ids'   ,				[additional_special_token_id]   )
     self.assertListEqual(getattr(lowerCAmelCase__   ,				'additional_special_tokens'   )   ,				[additional_special_token]   )
     self.assertListEqual(getattr(lowerCAmelCase__   ,				'additional_special_tokens_ids'   )   ,				[additional_special_token_id]   )







   def 		UpperCamelCase__      (						self   )					->     Dict:
     pass







   def 		UpperCamelCase__      (						self   )					->     List[str]:
     pass







   def 		UpperCamelCase__      (						self   )					->     Dict:
     pass







   def 		UpperCamelCase__      (						self   )					->     Dict:
     pass







   def 		UpperCamelCase__      (						self   )					->     Union[str, Any]:
     pass







   def 		UpperCamelCase__      (						self   )					->     str:
     pass







   def 		UpperCamelCase__      (						self   )					->     Any:
     pass







   def 		UpperCamelCase__      (						self   )					->     Optional[Any]:
     pass







import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class 	__lowercase    (__SCREAMING_SNAKE_CASE  ):





							"""simple docstring"""







							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Tuple					=					tempfile.mkdtemp()
										SCREAMING_SNAKE_CASE_		:     Dict					=					5

										# Realm tok
										SCREAMING_SNAKE_CASE_		:     Dict					=					[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'test',
										    'question',
										    'this',
										    'is',
										    'the',
										    'first',
										    'second',
										    'third',
										    'fourth',
										    'fifth',
										    'record',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										SCREAMING_SNAKE_CASE_		:     Tuple					=					os.path.join(self.tmpdirname			,						'realm_tokenizer'       )
										os.makedirs(lowerCAmelCase__			,						exist_ok=lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Tuple					=					os.path.join(lowerCAmelCase__			,						VOCAB_FILES_NAMES['vocab_file']       )
										with open(self.vocab_file			,						'w'			,						encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )

										SCREAMING_SNAKE_CASE_		:     str					=					os.path.join(self.tmpdirname			,						'realm_block_records'       )
										os.makedirs(lowerCAmelCase__			,						exist_ok=lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname			,						'realm_tokenizer'       )       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										shutil.rmtree(self.tmpdirname       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					RealmConfig(num_block_records=self.num_block_records       )
										return config






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					Dataset.from_dict(
										    {
										        'id': ['0', '1'],
										        'question': ['foo', 'bar'],
										        'answers': [['Foo', 'Bar'], ['Bar']],
										    }       )
										return dataset






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					np.array(
										    [
										        b'This is the first record',
										        b'This is the second record',
										        b'This is the third record',
										        b'This is the fourth record',
										        b'This is the fifth record',
										        b'This is a longer longer longer record',
										    ]			,						dtype=lowerCAmelCase__			,						)
										return block_records






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					RealmRetriever(
										    block_records=self.get_dummy_block_records()			,						tokenizer=self.get_tokenizer()			,						)
										return retriever






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					self.get_config()
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.get_dummy_retriever()
										SCREAMING_SNAKE_CASE_		:     List[str]					=					retriever.tokenizer

										SCREAMING_SNAKE_CASE_		:     Any					=					np.array([0, 3]			,						dtype='long'       )
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					tokenizer(['Test question']       ).input_ids
										SCREAMING_SNAKE_CASE_		:     int					=					tokenizer(
										    ['the fourth']			,						add_special_tokens=lowerCAmelCase__			,						return_token_type_ids=lowerCAmelCase__			,						return_attention_mask=lowerCAmelCase__			,						).input_ids
										SCREAMING_SNAKE_CASE_		:     Any					=					config.reader_seq_len

										SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     str					=					retriever(
										    lowerCAmelCase__			,						lowerCAmelCase__			,						answer_ids=lowerCAmelCase__			,						max_length=lowerCAmelCase__			,						return_tensors='np'       )

										self.assertEqual(len(lowerCAmelCase__       )			,						2       )
										self.assertEqual(len(lowerCAmelCase__       )			,						2       )
										self.assertEqual(len(lowerCAmelCase__       )			,						2       )
										self.assertEqual(concat_inputs.input_ids.shape			,						(2, 1_0)       )
										self.assertEqual(concat_inputs.attention_mask.shape			,						(2, 1_0)       )
										self.assertEqual(concat_inputs.token_type_ids.shape			,						(2, 1_0)       )
										self.assertEqual(concat_inputs.special_tokens_mask.shape			,						(2, 1_0)       )
										self.assertEqual(
										    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]       )			,						['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']			,						)
										self.assertEqual(
										    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]       )			,						['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']			,						)






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_config()
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					self.get_dummy_retriever()
										SCREAMING_SNAKE_CASE_		:     Tuple					=					retriever.tokenizer

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					np.array([0, 3, 5]			,						dtype='long'       )
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					tokenizer(['Test question']       ).input_ids
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer(
										    ['the fourth', 'longer longer']			,						add_special_tokens=lowerCAmelCase__			,						return_token_type_ids=lowerCAmelCase__			,						return_attention_mask=lowerCAmelCase__			,						).input_ids
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					config.reader_seq_len

										SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     Tuple					=					retriever(
										    lowerCAmelCase__			,						lowerCAmelCase__			,						answer_ids=lowerCAmelCase__			,						max_length=lowerCAmelCase__			,						return_tensors='np'       )

										self.assertEqual([False, True, True]			,						lowerCAmelCase__       )
										self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]			,						lowerCAmelCase__       )
										self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_dummy_retriever()
										retriever.save_pretrained(os.path.join(self.tmpdirname			,						'realm_block_records'       )       )

										# Test local path
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					retriever.from_pretrained(os.path.join(self.tmpdirname			,						'realm_block_records'       )       )
										self.assertEqual(retriever.block_records[0]			,						b'This is the first record'       )

										# Test mocked remote path
										with patch('transformers.models.realm.retrieval_realm.hf_hub_download'       ) as mock_hf_hub_download:
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					os.path.join(
													    os.path.join(self.tmpdirname			,						'realm_block_records'       )			,						_REALM_BLOCK_RECORDS_FILENAME       )
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa'       )

										self.assertEqual(retriever.block_records[0]			,						b'This is the first record'       )





