




import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer
class        __snake_case  (__UpperCAmelCase		):



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      tempfile.mkdtemp()
								_lowerCAmelCase    :    str										=      5

								# Realm tok
								_lowerCAmelCase    :    Union[str, Any]										=      [
								    """[UNK]""",
								    """[CLS]""",
								    """[SEP]""",
								    """[PAD]""",
								    """[MASK]""",
								    """test""",
								    """question""",
								    """this""",
								    """is""",
								    """the""",
								    """first""",
								    """second""",
								    """third""",
								    """fourth""",
								    """fifth""",
								    """record""",
								    """want""",
								    """##want""",
								    """##ed""",
								    """wa""",
								    """un""",
								    """runn""",
								    """##ing""",
								    """,""",
								    """low""",
								    """lowest""",
								]
								_lowerCAmelCase    :    Dict										=      os.path.join(self.tmpdirname		,	"""realm_tokenizer"""      )
								os.makedirs(_lowerCamelCase		,	exist_ok=_lowerCamelCase      )
								_lowerCAmelCase    :    Any										=      os.path.join(_lowerCamelCase		,	VOCAB_FILES_NAMES["""vocab_file"""]      )
								with open(self.vocab_file		,	"""w"""		,	encoding="""utf-8"""      ) as vocab_writer:
												vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )

								_lowerCAmelCase    :    Optional[Any]										=      os.path.join(self.tmpdirname		,	"""realm_block_records"""      )
								os.makedirs(_lowerCamelCase		,	exist_ok=_lowerCamelCase      )



				def       SCREAMING_SNAKE_CASE       (      self      :		str      )  ->    Any:





								'''simple docstring'''




								return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname		,	"""realm_tokenizer"""      )      )



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    Any:





								'''simple docstring'''




								shutil.rmtree(self.tmpdirname      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    List[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Any										=      RealmConfig(num_block_records=self.num_block_records      )
								return config



				def       SCREAMING_SNAKE_CASE       (      self      :		Optional[Any]      )  ->    List[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[Any]										=      Dataset.from_dict(
								    {
								        """id""": ["""0""", """1"""],
								        """question""": ["""foo""", """bar"""],
								        """answers""": [["""Foo""", """Bar"""], ["""Bar"""]],
								    }      )
								return dataset



				def       SCREAMING_SNAKE_CASE       (      self      :		List[Any]      )  ->    str:





								'''simple docstring'''




								_lowerCAmelCase    :    Union[str, Any]										=      np.array(
								    [
								        B"""This is the first record""",
								        B"""This is the second record""",
								        B"""This is the third record""",
								        B"""This is the fourth record""",
								        B"""This is the fifth record""",
								        B"""This is a longer longer longer record""",
								    ]		,	dtype=_lowerCamelCase		,	)
								return block_records



				def       SCREAMING_SNAKE_CASE       (      self      :		str      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[Any]										=      RealmRetriever(
								    block_records=self.get_dummy_block_records()		,	tokenizer=self.get_tokenizer()		,	)
								return retriever



				def       SCREAMING_SNAKE_CASE       (      self      :		Any      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[Any]										=      self.get_config()
								_lowerCAmelCase    :    Optional[Any]										=      self.get_dummy_retriever()
								_lowerCAmelCase    :    Tuple										=      retriever.tokenizer

								_lowerCAmelCase    :    int										=      np.array([0, 3]		,	dtype="""long"""      )
								_lowerCAmelCase    :    str										=      tokenizer(["""Test question"""]      ).input_ids
								_lowerCAmelCase    :    List[Any]										=      tokenizer(
								    ["""the fourth"""]		,	add_special_tokens=_lowerCamelCase		,	return_token_type_ids=_lowerCamelCase		,	return_attention_mask=_lowerCamelCase		,	).input_ids
								_lowerCAmelCase    :    Optional[int]										=      config.reader_seq_len

								_lowerCAmelCase					,    _lowerCAmelCase					,    _lowerCAmelCase					,    _lowerCAmelCase    :    Optional[int]										=      retriever(
								    _lowerCamelCase		,	_lowerCamelCase		,	answer_ids=_lowerCamelCase		,	max_length=_lowerCamelCase		,	return_tensors="""np"""      )

								self.assertEqual(len(_lowerCamelCase      )		,	2      )
								self.assertEqual(len(_lowerCamelCase      )		,	2      )
								self.assertEqual(len(_lowerCamelCase      )		,	2      )
								self.assertEqual(concat_inputs.input_ids.shape		,	(2, 10)      )
								self.assertEqual(concat_inputs.attention_mask.shape		,	(2, 10)      )
								self.assertEqual(concat_inputs.token_type_ids.shape		,	(2, 10)      )
								self.assertEqual(concat_inputs.special_tokens_mask.shape		,	(2, 10)      )
								self.assertEqual(
								    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]      )		,	["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """first""", """record""", """[SEP]"""]		,	)
								self.assertEqual(
								    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]      )		,	["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """fourth""", """record""", """[SEP]"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    int:





								'''simple docstring'''




								_lowerCAmelCase    :    Any										=      self.get_config()
								_lowerCAmelCase    :    Tuple										=      self.get_dummy_retriever()
								_lowerCAmelCase    :    Dict										=      retriever.tokenizer

								_lowerCAmelCase    :    Tuple										=      np.array([0, 3, 5]		,	dtype="""long"""      )
								_lowerCAmelCase    :    int										=      tokenizer(["""Test question"""]      ).input_ids
								_lowerCAmelCase    :    int										=      tokenizer(
								    ["""the fourth""", """longer longer"""]		,	add_special_tokens=_lowerCamelCase		,	return_token_type_ids=_lowerCamelCase		,	return_attention_mask=_lowerCamelCase		,	).input_ids
								_lowerCAmelCase    :    List[Any]										=      config.reader_seq_len

								_lowerCAmelCase					,    _lowerCAmelCase					,    _lowerCAmelCase					,    _lowerCAmelCase    :    Tuple										=      retriever(
								    _lowerCamelCase		,	_lowerCamelCase		,	answer_ids=_lowerCamelCase		,	max_length=_lowerCamelCase		,	return_tensors="""np"""      )

								self.assertEqual([False, True, True]		,	_lowerCamelCase      )
								self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]		,	_lowerCamelCase      )
								self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]		,	_lowerCamelCase      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Union[str, Any]      )  ->    List[str]:





								'''simple docstring'''




								_lowerCAmelCase    :    Union[str, Any]										=      self.get_dummy_retriever()
								retriever.save_pretrained(os.path.join(self.tmpdirname		,	"""realm_block_records"""      )      )

								# Test local path
								_lowerCAmelCase    :    Any										=      retriever.from_pretrained(os.path.join(self.tmpdirname		,	"""realm_block_records"""      )      )
								self.assertEqual(retriever.block_records[0]		,	B"""This is the first record"""      )

								# Test mocked remote path
								with patch("""transformers.models.realm.retrieval_realm.hf_hub_download"""      ) as mock_hf_hub_download:
												_lowerCAmelCase    :    Optional[Any]										=      os.path.join(
												    os.path.join(self.tmpdirname		,	"""realm_block_records"""      )		,	_REALM_BLOCK_RECORDS_FILENAME      )
												_lowerCAmelCase    :    Tuple										=      RealmRetriever.from_pretrained("""google/realm-cc-news-pretrained-openqa"""      )

								self.assertEqual(retriever.block_records[0]		,	B"""This is the first record"""      )







def 							_UpperCAmelCase  (UpperCamelCase_ : int       ):
				'''simple docstring'''
				_lowerCAmelCase    :    int										=      n ** (1 / 3)
				return (val * val * val) == n


if __name__ == "__main__":
	print(perfect_cube(2_7))
	print(perfect_cube(4))

