


'''simple docstring'''
import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer
class      lowercase			(							__lowerCAmelCase      ):

		"""simple docstring"""

		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				tempfile.mkdtemp()
							UpperCamelCase__     :Union[str, Any]							      =				5

							# Realm tok
							UpperCamelCase__     :int							      =				[
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''test''',
							    '''question''',
							    '''this''',
							    '''is''',
							    '''the''',
							    '''first''',
							    '''second''',
							    '''third''',
							    '''fourth''',
							    '''fifth''',
							    '''record''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							UpperCamelCase__     :str							      =				os.path.join(self.tmpdirname					,     '''realm_tokenizer'''							)
							os.makedirs(lowerCAmelCase_					,     exist_ok=lowerCAmelCase_							)
							UpperCamelCase__     :List[Any]							      =				os.path.join(lowerCAmelCase_					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)

							UpperCamelCase__     :List[str]							      =				os.path.join(self.tmpdirname					,     '''realm_block_records'''							)
							os.makedirs(lowerCAmelCase_					,     exist_ok=lowerCAmelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,     '''realm_tokenizer'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							shutil.rmtree(self.tmpdirname							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RealmConfig(num_block_records=self.num_block_records							)
							return config




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				Dataset.from_dict(
							    {
							        '''id''': ['''0''', '''1'''],
							        '''question''': ['''foo''', '''bar'''],
							        '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
							    }							)
							return dataset




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				np.array(
							    [
							        b'''This is the first record''',
							        b'''This is the second record''',
							        b'''This is the third record''',
							        b'''This is the fourth record''',
							        b'''This is the fifth record''',
							        b'''This is a longer longer longer record''',
							    ]					,     dtype=lowerCAmelCase_					,     )
							return block_records




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				RealmRetriever(
							    block_records=self.get_dummy_block_records()					,     tokenizer=self.get_tokenizer()					,     )
							return retriever




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.get_config()
							UpperCamelCase__     :List[str]							      =				self.get_dummy_retriever()
							UpperCamelCase__     :Any							      =				retriever.tokenizer

							UpperCamelCase__     :Tuple							      =				np.array([0, 3]					,     dtype='''long'''							)
							UpperCamelCase__     :Optional[Any]							      =				tokenizer(['''Test question''']							).input_ids
							UpperCamelCase__     :Any							      =				tokenizer(
							    ['''the fourth''']					,     add_special_tokens=lowerCAmelCase_					,     return_token_type_ids=lowerCAmelCase_					,     return_attention_mask=lowerCAmelCase_					,     ).input_ids
							UpperCamelCase__     :List[str]							      =				config.reader_seq_len

							UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :Optional[int]							      =				retriever(
							    lowerCAmelCase_					,     lowerCAmelCase_					,     answer_ids=lowerCAmelCase_					,     max_length=lowerCAmelCase_					,     return_tensors='''np'''							)

							self.assertEqual(len(lowerCAmelCase_							)					,     2							)
							self.assertEqual(len(lowerCAmelCase_							)					,     2							)
							self.assertEqual(len(lowerCAmelCase_							)					,     2							)
							self.assertEqual(concat_inputs.input_ids.shape					,     (2, 10)							)
							self.assertEqual(concat_inputs.attention_mask.shape					,     (2, 10)							)
							self.assertEqual(concat_inputs.token_type_ids.shape					,     (2, 10)							)
							self.assertEqual(concat_inputs.special_tokens_mask.shape					,     (2, 10)							)
							self.assertEqual(
							    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]							)					,     ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']					,     )
							self.assertEqual(
							    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]							)					,     ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']					,     )




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				self.get_config()
							UpperCamelCase__     :Any							      =				self.get_dummy_retriever()
							UpperCamelCase__     :Dict							      =				retriever.tokenizer

							UpperCamelCase__     :Optional[Any]							      =				np.array([0, 3, 5]					,     dtype='''long'''							)
							UpperCamelCase__     :List[Any]							      =				tokenizer(['''Test question''']							).input_ids
							UpperCamelCase__     :Union[str, Any]							      =				tokenizer(
							    ['''the fourth''', '''longer longer''']					,     add_special_tokens=lowerCAmelCase_					,     return_token_type_ids=lowerCAmelCase_					,     return_attention_mask=lowerCAmelCase_					,     ).input_ids
							UpperCamelCase__     :Tuple							      =				config.reader_seq_len

							UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :str							      =				retriever(
							    lowerCAmelCase_					,     lowerCAmelCase_					,     answer_ids=lowerCAmelCase_					,     max_length=lowerCAmelCase_					,     return_tensors='''np'''							)

							self.assertEqual([False, True, True]					,     lowerCAmelCase_							)
							self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,     lowerCAmelCase_							)
							self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,     lowerCAmelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				self.get_dummy_retriever()
							retriever.save_pretrained(os.path.join(self.tmpdirname					,     '''realm_block_records'''							)							)

							# Test local path
							UpperCamelCase__     :Any							      =				retriever.from_pretrained(os.path.join(self.tmpdirname					,     '''realm_block_records'''							)							)
							self.assertEqual(retriever.block_records[0]					,     b'''This is the first record'''							)

							# Test mocked remote path
							with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''							) as mock_hf_hub_download:
												UpperCamelCase__     :Any							      =				os.path.join(
												    os.path.join(self.tmpdirname					,     '''realm_block_records'''							)					,     _REALM_BLOCK_RECORDS_FILENAME							)
												UpperCamelCase__     :int							      =				RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''							)

							self.assertEqual(retriever.block_records[0]					,     b'''This is the first record'''							)



'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


__snake_case							   =  {
    '''configuration_layoutlmv3''': [
        '''LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''LayoutLMv3Config''',
        '''LayoutLMv3OnnxConfig''',
    ],
    '''processing_layoutlmv3''': ['''LayoutLMv3Processor'''],
    '''tokenization_layoutlmv3''': ['''LayoutLMv3Tokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  ['''LayoutLMv3TokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  [
		    '''LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''LayoutLMv3ForQuestionAnswering''',
		    '''LayoutLMv3ForSequenceClassification''',
		    '''LayoutLMv3ForTokenClassification''',
		    '''LayoutLMv3Model''',
		    '''LayoutLMv3PreTrainedModel''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  [
		    '''TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFLayoutLMv3ForQuestionAnswering''',
		    '''TFLayoutLMv3ForSequenceClassification''',
		    '''TFLayoutLMv3ForTokenClassification''',
		    '''TFLayoutLMv3Model''',
		    '''TFLayoutLMv3PreTrainedModel''',
		]

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  ['''LayoutLMv3FeatureExtractor''']
		__snake_case							   =  ['''LayoutLMv3ImageProcessor''']


if TYPE_CHECKING:
		from .configuration_layoutlmva import (
		    LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    LayoutLMvaConfig,
		    LayoutLMvaOnnxConfig,
		)
		from .processing_layoutlmva import LayoutLMvaProcessor
		from .tokenization_layoutlmva import LayoutLMvaTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_layoutlmva import (
				    LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LayoutLMvaForQuestionAnswering,
				    LayoutLMvaForSequenceClassification,
				    LayoutLMvaForTokenClassification,
				    LayoutLMvaModel,
				    LayoutLMvaPreTrainedModel,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_layoutlmva import (
				    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFLayoutLMvaForQuestionAnswering,
				    TFLayoutLMvaForSequenceClassification,
				    TFLayoutLMvaForTokenClassification,
				    TFLayoutLMvaModel,
				    TFLayoutLMvaPreTrainedModel,
				)

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
				from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
		import sys

		__snake_case							   =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)