





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer

class      A       (	__lowercase						):



      def 	lowerCAmelCase__    (	self:						Union[str, Any]       )						->    Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_											=tempfile.mkdtemp()
         UpperCAmelCase_											=5

         # Realm tok
         UpperCAmelCase_											=[
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "test",
             "question",
             "this",
             "is",
             "the",
             "first",
             "second",
             "third",
             "fourth",
             "fifth",
             "record",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         UpperCAmelCase_											=os.path.join(self.tmpdirname		,  "realm_tokenizer"       )
         os.makedirs(_lowerCAmelCase		,  exist_ok=_lowerCAmelCase       )
         UpperCAmelCase_											=os.path.join(_lowerCAmelCase		,  VOCAB_FILES_NAMES["vocab_file"]       )
         with open(self.vocab_file		,  "w"		,  encoding="utf-8"       ) as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )

         UpperCAmelCase_											=os.path.join(self.tmpdirname		,  "realm_block_records"       )
         os.makedirs(_lowerCAmelCase		,  exist_ok=_lowerCAmelCase       )



      def 	lowerCAmelCase__    (	self:						Dict       )						->    RealmTokenizer:
         '''simple docstring'''

         return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname		,  "realm_tokenizer"       )       )



      def 	lowerCAmelCase__    (	self:						Optional[int]       )						->    List[str]:
         '''simple docstring'''

         shutil.rmtree(self.tmpdirname       )



      def 	lowerCAmelCase__    (	self:						Tuple       )						->    Dict:
         '''simple docstring'''

         UpperCAmelCase_											=RealmConfig(num_block_records=self.num_block_records       )
         return config



      def 	lowerCAmelCase__    (	self:						int       )						->    Tuple:
         '''simple docstring'''

         UpperCAmelCase_											=Dataset.from_dict(
             {
                 "id": ["0", "1"],
                 "question": ["foo", "bar"],
                 "answers": [["Foo", "Bar"], ["Bar"]],
             }       )
         return dataset



      def 	lowerCAmelCase__    (	self:						Any       )						->    int:
         '''simple docstring'''

         UpperCAmelCase_											=np.array(
             [
                 b"This is the first record",
                 b"This is the second record",
                 b"This is the third record",
                 b"This is the fourth record",
                 b"This is the fifth record",
                 b"This is a longer longer longer record",
             ]		,  dtype=_lowerCAmelCase		,  )
         return block_records



      def 	lowerCAmelCase__    (	self:						str       )						->    Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_											=RealmRetriever(
             block_records=self.get_dummy_block_records()		,  tokenizer=self.get_tokenizer()		,  )
         return retriever



      def 	lowerCAmelCase__    (	self:						Union[str, Any]       )						->    Dict:
         '''simple docstring'''

         UpperCAmelCase_											=self.get_config()
         UpperCAmelCase_											=self.get_dummy_retriever()
         UpperCAmelCase_											=retriever.tokenizer

         UpperCAmelCase_											=np.array([0, 3]		,  dtype="long"       )
         UpperCAmelCase_											=tokenizer(["Test question"]       ).input_ids
         UpperCAmelCase_											=tokenizer(
             ["the fourth"]		,  add_special_tokens=_lowerCAmelCase		,  return_token_type_ids=_lowerCAmelCase		,  return_attention_mask=_lowerCAmelCase		,  ).input_ids
         UpperCAmelCase_											=config.reader_seq_len

         UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_											=retriever(
             _lowerCAmelCase		,  _lowerCAmelCase		,  answer_ids=_lowerCAmelCase		,  max_length=_lowerCAmelCase		,  return_tensors="np"       )

         self.assertEqual(len(_lowerCAmelCase       )		,  2       )
         self.assertEqual(len(_lowerCAmelCase       )		,  2       )
         self.assertEqual(len(_lowerCAmelCase       )		,  2       )
         self.assertEqual(concat_inputs.input_ids.shape		,  (2, 10)       )
         self.assertEqual(concat_inputs.attention_mask.shape		,  (2, 10)       )
         self.assertEqual(concat_inputs.token_type_ids.shape		,  (2, 10)       )
         self.assertEqual(concat_inputs.special_tokens_mask.shape		,  (2, 10)       )
         self.assertEqual(
             tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]       )		,  ["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "first", "record", "[SEP]"]		,  )
         self.assertEqual(
             tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]       )		,  ["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "fourth", "record", "[SEP]"]		,  )



      def 	lowerCAmelCase__    (	self:						int       )						->    Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_											=self.get_config()
         UpperCAmelCase_											=self.get_dummy_retriever()
         UpperCAmelCase_											=retriever.tokenizer

         UpperCAmelCase_											=np.array([0, 3, 5]		,  dtype="long"       )
         UpperCAmelCase_											=tokenizer(["Test question"]       ).input_ids
         UpperCAmelCase_											=tokenizer(
             ["the fourth", "longer longer"]		,  add_special_tokens=_lowerCAmelCase		,  return_token_type_ids=_lowerCAmelCase		,  return_attention_mask=_lowerCAmelCase		,  ).input_ids
         UpperCAmelCase_											=config.reader_seq_len

         UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_											=retriever(
             _lowerCAmelCase		,  _lowerCAmelCase		,  answer_ids=_lowerCAmelCase		,  max_length=_lowerCAmelCase		,  return_tensors="np"       )

         self.assertEqual([False, True, True]		,  _lowerCAmelCase       )
         self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]		,  _lowerCAmelCase       )
         self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]		,  _lowerCAmelCase       )






      def 	lowerCAmelCase__    (	self:						Optional[Any]       )						->    List[Any]:
         '''simple docstring'''

         UpperCAmelCase_											=self.get_dummy_retriever()
         retriever.save_pretrained(os.path.join(self.tmpdirname		,  "realm_block_records"       )       )

         # Test local path
         UpperCAmelCase_											=retriever.from_pretrained(os.path.join(self.tmpdirname		,  "realm_block_records"       )       )
         self.assertEqual(retriever.block_records[0]		,  b"This is the first record"       )

         # Test mocked remote path
         with patch("transformers.models.realm.retrieval_realm.hf_hub_download"       ) as mock_hf_hub_download:
            UpperCAmelCase_											=os.path.join(
                os.path.join(self.tmpdirname		,  "realm_block_records"       )		,  _REALM_BLOCK_RECORDS_FILENAME       )
            UpperCAmelCase_											=RealmRetriever.from_pretrained("google/realm-cc-news-pretrained-openqa"       )

         self.assertEqual(retriever.block_records[0]		,  b"This is the first record"       )











from ....configuration_utils import PretrainedConfig
from ....utils import logging


__lowercase  :   Optional[int]						      =logging.get_logger(__name__)

__lowercase  :   Union[str, Any]						      ={
    """Visual-Attention-Network/van-base""": (
        """https://huggingface.co/Visual-Attention-Network/van-base/blob/main/config.json"""
    ),
}

class      A       (	__lowercase						):
      _snake_case							     ='''van'''



      def __init__(	self:						str		,  _lowerCAmelCase:						List[Any]=224		,  _lowerCAmelCase:						Any=3		,  _lowerCAmelCase:						Tuple=[7, 3, 3, 3]		,  _lowerCAmelCase:						Union[str, Any]=[4, 2, 2, 2]		,  _lowerCAmelCase:						List[str]=[64, 128, 320, 512]		,  _lowerCAmelCase:						Tuple=[3, 3, 12, 3]		,  _lowerCAmelCase:						Union[str, Any]=[8, 8, 4, 4]		,  _lowerCAmelCase:						Any="gelu"		,  _lowerCAmelCase:						Optional[int]=0.02		,  _lowerCAmelCase:						List[str]=1e-6		,  _lowerCAmelCase:						Any=1e-2		,  _lowerCAmelCase:						Any=0.0		,  _lowerCAmelCase:						Optional[Any]=0.0		,  **_lowerCAmelCase:						List[str]		,  )						->    Dict:
         '''simple docstring'''

         super().__init__(**_lowerCAmelCase       )
         UpperCAmelCase_											=image_size
         UpperCAmelCase_											=num_channels
         UpperCAmelCase_											=patch_sizes
         UpperCAmelCase_											=strides
         UpperCAmelCase_											=hidden_sizes
         UpperCAmelCase_											=depths
         UpperCAmelCase_											=mlp_ratios
         UpperCAmelCase_											=hidden_act
         UpperCAmelCase_											=initializer_range
         UpperCAmelCase_											=layer_norm_eps
         UpperCAmelCase_											=layer_scale_init_value
         UpperCAmelCase_											=drop_path_rate
         UpperCAmelCase_											=dropout_rate




