

"""simple docstring"""




import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class      __lowercase			(    __lowerCamelCase					):

  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					tempfile.mkdtemp()
   UpperCAmelCase__					:  Optional[Any]					   =					5

   # Realm tok
   UpperCAmelCase__					:  List[Any]					   =					[
       """[UNK]""",
       """[CLS]""",
       """[SEP]""",
       """[PAD]""",
       """[MASK]""",
       """test""",
       """question""",
       """this""",
       """is""",
       """the""",
       """first""",
       """second""",
       """third""",
       """fourth""",
       """fifth""",
       """record""",
       """want""",
       """##want""",
       """##ed""",
       """wa""",
       """un""",
       """runn""",
       """##ing""",
       """,""",
       """low""",
       """lowest""",
   ]
   UpperCAmelCase__					:  str					   =					os.path.join(self.tmpdirname					,"""realm_tokenizer"""    )
   os.makedirs(A					,exist_ok=A    )
   UpperCAmelCase__					:  Optional[int]					   =					os.path.join(A					,VOCAB_FILES_NAMES["""vocab_file"""]    )
   with open(self.vocab_file					,"""w"""					,encoding="""utf-8"""    ) as vocab_writer:
    vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )

   UpperCAmelCase__					:  Optional[int]					   =					os.path.join(self.tmpdirname					,"""realm_block_records"""    )
   os.makedirs(A					,exist_ok=A    )

  def    __lowercase       (  self		:			List[Any]    ):






   '''simple docstring'''
   return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,"""realm_tokenizer"""    )    )

  def    __lowercase       (  self		:			List[str]    ):






   '''simple docstring'''
   shutil.rmtree(self.tmpdirname    )

  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Optional[int]					   =					RealmConfig(num_block_records=self.num_block_records    )
   return config

  def    __lowercase       (  self		:			Optional[Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					Dataset.from_dict(
       {
           """id""": ["""0""", """1"""],
           """question""": ["""foo""", """bar"""],
           """answers""": [["""Foo""", """Bar"""], ["""Bar"""]],
       }    )
   return dataset

  def    __lowercase       (  self		:			str    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Any					   =					np.array(
       [
           B"""This is the first record""",
           B"""This is the second record""",
           B"""This is the third record""",
           B"""This is the fourth record""",
           B"""This is the fifth record""",
           B"""This is a longer longer longer record""",
       ]					,dtype=A					,)
   return block_records

  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					RealmRetriever(
       block_records=self.get_dummy_block_records()					,tokenizer=self.get_tokenizer()					,)
   return retriever

  def    __lowercase       (  self		:			Tuple    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Any					   =					self.get_config()
   UpperCAmelCase__					:  List[Any]					   =					self.get_dummy_retriever()
   UpperCAmelCase__					:  List[Any]					   =					retriever.tokenizer

   UpperCAmelCase__					:  List[Any]					   =					np.array([0, 3]					,dtype="""long"""    )
   UpperCAmelCase__					:  List[str]					   =					tokenizer(["""Test question"""]    ).input_ids
   UpperCAmelCase__					:  Any					   =					tokenizer(
       ["""the fourth"""]					,add_special_tokens=A					,return_token_type_ids=A					,return_attention_mask=A					,).input_ids
   UpperCAmelCase__					:  Union[str, Any]					   =					config.reader_seq_len

   UpperCAmelCase__   ,      UpperCAmelCase__   ,      UpperCAmelCase__   ,      UpperCAmelCase__					:  List[Any]					   =					retriever(
       A					,A					,answer_ids=A					,max_length=A					,return_tensors="""np"""    )

   self.assertEqual(len(A    )					,2    )
   self.assertEqual(len(A    )					,2    )
   self.assertEqual(len(A    )					,2    )
   self.assertEqual(concat_inputs.input_ids.shape					,(2, 10)    )
   self.assertEqual(concat_inputs.attention_mask.shape					,(2, 10)    )
   self.assertEqual(concat_inputs.token_type_ids.shape					,(2, 10)    )
   self.assertEqual(concat_inputs.special_tokens_mask.shape					,(2, 10)    )
   self.assertEqual(
       tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]    )					,["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """first""", """record""", """[SEP]"""]					,)
   self.assertEqual(
       tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]    )					,["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """fourth""", """record""", """[SEP]"""]					,)

  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					self.get_config()
   UpperCAmelCase__					:  List[Any]					   =					self.get_dummy_retriever()
   UpperCAmelCase__					:  Dict					   =					retriever.tokenizer

   UpperCAmelCase__					:  Union[str, Any]					   =					np.array([0, 3, 5]					,dtype="""long"""    )
   UpperCAmelCase__					:  Optional[int]					   =					tokenizer(["""Test question"""]    ).input_ids
   UpperCAmelCase__					:  List[Any]					   =					tokenizer(
       ["""the fourth""", """longer longer"""]					,add_special_tokens=A					,return_token_type_ids=A					,return_attention_mask=A					,).input_ids
   UpperCAmelCase__					:  Tuple					   =					config.reader_seq_len

   UpperCAmelCase__   ,      UpperCAmelCase__   ,      UpperCAmelCase__   ,      UpperCAmelCase__					:  Tuple					   =					retriever(
       A					,A					,answer_ids=A					,max_length=A					,return_tensors="""np"""    )

   self.assertEqual([False, True, True]					,A    )
   self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,A    )
   self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,A    )




  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Any					   =					self.get_dummy_retriever()
   retriever.save_pretrained(os.path.join(self.tmpdirname					,"""realm_block_records"""    )    )

   # Test local path
   UpperCAmelCase__					:  int					   =					retriever.from_pretrained(os.path.join(self.tmpdirname					,"""realm_block_records"""    )    )
   self.assertEqual(retriever.block_records[0]					,B"""This is the first record"""    )

   # Test mocked remote path
   with patch("""transformers.models.realm.retrieval_realm.hf_hub_download"""    ) as mock_hf_hub_download:
    UpperCAmelCase__					:  List[Any]					   =					os.path.join(
        os.path.join(self.tmpdirname					,"""realm_block_records"""    )					,_REALM_BLOCK_RECORDS_FILENAME    )
    UpperCAmelCase__					:  Optional[int]					   =					RealmRetriever.from_pretrained("""google/realm-cc-news-pretrained-openqa"""    )

   self.assertEqual(retriever.block_records[0]					,B"""This is the first record"""    )








"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase =				logging.get_logger(__name__)

__UpperCAmelCase =				{
    'microsoft/markuplm-base': 'https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json',
    'microsoft/markuplm-large': 'https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json',
}







class      __lowercase			(    __lowerCamelCase					):
  snake_case_				       =       """markuplm"""

  def __init__(  self		:			str					,A		:			List[Any]=30_522					,A		:			Tuple=768					,A		:			str=12					,A		:			int=12					,A		:			int=3_072					,A		:			Optional[int]="gelu"					,A		:			Optional[int]=0.1					,A		:			Optional[int]=0.1					,A		:			Any=512					,A		:			Any=2					,A		:			str=0.0_2					,A		:			int=1e-12					,A		:			int=0					,A		:			str=0					,A		:			List[Any]=2					,A		:			List[str]=256					,A		:			Union[str, Any]=1_024					,A		:			List[Any]=216					,A		:			Any=1_001					,A		:			Optional[int]=32					,A		:			Any=50					,A		:			int="absolute"					,A		:			Dict=True					,A		:			int=None					,**A		:			Optional[int]					,):






   '''simple docstring'''
   super().__init__(
       pad_token_id=A					,bos_token_id=A					,eos_token_id=A					,**A					,)
   UpperCAmelCase__					:  Dict					   =					vocab_size
   UpperCAmelCase__					:  Union[str, Any]					   =					hidden_size
   UpperCAmelCase__					:  Optional[int]					   =					num_hidden_layers
   UpperCAmelCase__					:  Tuple					   =					num_attention_heads
   UpperCAmelCase__					:  Optional[Any]					   =					hidden_act
   UpperCAmelCase__					:  Any					   =					intermediate_size
   UpperCAmelCase__					:  Dict					   =					hidden_dropout_prob
   UpperCAmelCase__					:  Optional[Any]					   =					attention_probs_dropout_prob
   UpperCAmelCase__					:  Optional[int]					   =					max_position_embeddings
   UpperCAmelCase__					:  Dict					   =					type_vocab_size
   UpperCAmelCase__					:  Optional[Any]					   =					initializer_range
   UpperCAmelCase__					:  str					   =					layer_norm_eps
   UpperCAmelCase__					:  Any					   =					position_embedding_type
   UpperCAmelCase__					:  int					   =					use_cache
   UpperCAmelCase__					:  List[str]					   =					classifier_dropout
   # additional properties
   UpperCAmelCase__					:  Optional[int]					   =					max_depth
   UpperCAmelCase__					:  List[str]					   =					max_xpath_tag_unit_embeddings
   UpperCAmelCase__					:  Any					   =					max_xpath_subs_unit_embeddings
   UpperCAmelCase__					:  str					   =					tag_pad_id
   UpperCAmelCase__					:  Dict					   =					subs_pad_id
   UpperCAmelCase__					:  List[str]					   =					xpath_unit_hidden_size





