





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer

class SCREAMING_SNAKE_CASE_ (					_lowerCAmelCase   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       tempfile.mkdtemp()
										lowerCamelCase__       =       5

										# Realm tok
										lowerCamelCase__       =       [
										    "[UNK]",
										    "[CLS]",
										    "[SEP]",
										    "[PAD]",
										    "[MASK]",
										    "test",
										    "question",
										    "this",
										    "is",
										    "the",
										    "first",
										    "second",
										    "third",
										    "fourth",
										    "fifth",
										    "record",
										    "want",
										    "##want",
										    "##ed",
										    "wa",
										    "un",
										    "runn",
										    "##ing",
										    ",",
										    "low",
										    "lowest",
										]
										lowerCamelCase__       =       os.path.join(self.tmpdirname	,  "realm_tokenizer"	)
										os.makedirs(_lowerCAmelCase	,  exist_ok=_lowerCAmelCase	)
										lowerCamelCase__       =       os.path.join(_lowerCAmelCase	,  VOCAB_FILES_NAMES["vocab_file"]	)
										with open(self.vocab_file	,  "w"	,  encoding="utf-8"	) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)

										lowerCamelCase__       =       os.path.join(self.tmpdirname	,  "realm_block_records"	)
										os.makedirs(_lowerCAmelCase	,  exist_ok=_lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname	,  "realm_tokenizer"	)	)





					def 					__magic_name__ ( self	):
										shutil.rmtree(self.tmpdirname	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       RealmConfig(num_block_records=self.num_block_records	)
										return config





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       Dataset.from_dict(
										    {
										        "id": ["0", "1"],
										        "question": ["foo", "bar"],
										        "answers": [["Foo", "Bar"], ["Bar"]],
										    }	)
										return dataset





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       np.array(
										    [
										        b"This is the first record",
										        b"This is the second record",
										        b"This is the third record",
										        b"This is the fourth record",
										        b"This is the fifth record",
										        b"This is a longer longer longer record",
										    ]	,  dtype=_lowerCAmelCase	,  )
										return block_records





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       RealmRetriever(
										    block_records=self.get_dummy_block_records()	,  tokenizer=self.get_tokenizer()	,  )
										return retriever





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_config()
										lowerCamelCase__       =       self.get_dummy_retriever()
										lowerCamelCase__       =       retriever.tokenizer

										lowerCamelCase__       =       np.array([0, 3]	,  dtype="long"	)
										lowerCamelCase__       =       tokenizer(["Test question"]	).input_ids
										lowerCamelCase__       =       tokenizer(
										    ["the fourth"]	,  add_special_tokens=_lowerCAmelCase	,  return_token_type_ids=_lowerCAmelCase	,  return_attention_mask=_lowerCAmelCase	,  ).input_ids
										lowerCamelCase__       =       config.reader_seq_len

										lowerCamelCase__    ,		lowerCamelCase__    ,		lowerCamelCase__    ,		lowerCamelCase__       =       retriever(
										    _lowerCAmelCase	,  _lowerCAmelCase	,  answer_ids=_lowerCAmelCase	,  max_length=_lowerCAmelCase	,  return_tensors="np"	)

										self.assertEqual(len(_lowerCAmelCase	)	,  2	)
										self.assertEqual(len(_lowerCAmelCase	)	,  2	)
										self.assertEqual(len(_lowerCAmelCase	)	,  2	)
										self.assertEqual(concat_inputs.input_ids.shape	,  (2, 10)	)
										self.assertEqual(concat_inputs.attention_mask.shape	,  (2, 10)	)
										self.assertEqual(concat_inputs.token_type_ids.shape	,  (2, 10)	)
										self.assertEqual(concat_inputs.special_tokens_mask.shape	,  (2, 10)	)
										self.assertEqual(
										    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]	)	,  ["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "first", "record", "[SEP]"]	,  )
										self.assertEqual(
										    tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]	)	,  ["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "fourth", "record", "[SEP]"]	,  )





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_config()
										lowerCamelCase__       =       self.get_dummy_retriever()
										lowerCamelCase__       =       retriever.tokenizer

										lowerCamelCase__       =       np.array([0, 3, 5]	,  dtype="long"	)
										lowerCamelCase__       =       tokenizer(["Test question"]	).input_ids
										lowerCamelCase__       =       tokenizer(
										    ["the fourth", "longer longer"]	,  add_special_tokens=_lowerCAmelCase	,  return_token_type_ids=_lowerCAmelCase	,  return_attention_mask=_lowerCAmelCase	,  ).input_ids
										lowerCamelCase__       =       config.reader_seq_len

										lowerCamelCase__    ,		lowerCamelCase__    ,		lowerCamelCase__    ,		lowerCamelCase__       =       retriever(
										    _lowerCAmelCase	,  _lowerCAmelCase	,  answer_ids=_lowerCAmelCase	,  max_length=_lowerCAmelCase	,  return_tensors="np"	)

										self.assertEqual([False, True, True]	,  _lowerCAmelCase	)
										self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]	,  _lowerCAmelCase	)
										self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]	,  _lowerCAmelCase	)






					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_dummy_retriever()
										retriever.save_pretrained(os.path.join(self.tmpdirname	,  "realm_block_records"	)	)

										# Test local path
										lowerCamelCase__       =       retriever.from_pretrained(os.path.join(self.tmpdirname	,  "realm_block_records"	)	)
										self.assertEqual(retriever.block_records[0]	,  b"This is the first record"	)

										# Test mocked remote path
										with patch("transformers.models.realm.retrieval_realm.hf_hub_download"	) as mock_hf_hub_download:
															lowerCamelCase__       =       os.path.join(
															    os.path.join(self.tmpdirname	,  "realm_block_records"	)	,  _REALM_BLOCK_RECORDS_FILENAME	)
															lowerCamelCase__       =       RealmRetriever.from_pretrained("google/realm-cc-news-pretrained-openqa"	)

										self.assertEqual(retriever.block_records[0]	,  b"This is the first record"	)












from __future__ import annotations

import math
import random
from collections.abc import Collection
from typing import overload

class SCREAMING_SNAKE_CASE_ :





					"""simple docstring"""


					def __init__( self	,  _lowerCAmelCase = None	):
										if components is None:
															lowerCamelCase__       =       []
										lowerCamelCase__       =       list(_lowerCAmelCase	)





					def __len__( self	):
										return len(self.__components	)





					def __str__( self	):
										return "(" + ",".join(map(_lowerCAmelCase	,  self.__components	)	) + ")"





					def __add__( self	,  _lowerCAmelCase	):
										lowerCamelCase__       =       len(self	)
										if size == len(_lowerCAmelCase	):
															lowerCamelCase__       =       [self.__components[i] + other.component(_lowerCAmelCase	) for i in range(_lowerCAmelCase	)]
															return Vector(_lowerCAmelCase	)
										else:
															raise Exception("must have the same size"	)





					def __sub__( self	,  _lowerCAmelCase	):
										lowerCamelCase__       =       len(self	)
										if size == len(_lowerCAmelCase	):
															lowerCamelCase__       =       [self.__components[i] - other.component(_lowerCAmelCase	) for i in range(_lowerCAmelCase	)]
															return Vector(_lowerCAmelCase	)
										else:  # error case
															raise Exception("must have the same size"	)





					@overload
					def __mul__( self	,  _lowerCAmelCase	):
										...





					@overload
					def __mul__( self	,  _lowerCAmelCase	):
										...





					def __mul__( self	,  _lowerCAmelCase	):
										if isinstance(_lowerCAmelCase	,  (float, int)	):
															lowerCamelCase__       =       [c * other for c in self.__components]
															return Vector(_lowerCAmelCase	)
										elif isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) and len(self	) == len(_lowerCAmelCase	):
															lowerCamelCase__       =       len(self	)
															lowerCamelCase__       =       [self.__components[i] * other.component(_lowerCAmelCase	) for i in range(_lowerCAmelCase	)]
															return sum(_lowerCAmelCase	)
										else:  # error case
															raise Exception("invalid operand!"	)





					def 					__magic_name__ ( self	):
										return Vector(self.__components	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) and -len(self.__components	) <= i < len(self.__components	):
															return self.__components[i]
										else:
															raise Exception("index out of range"	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										assert -len(self.__components	) <= pos < len(self.__components	)
										lowerCamelCase__       =       value





					def 					__magic_name__ ( self	):
										if len(self.__components	) == 0:
															raise Exception("Vector is empty"	)
										lowerCamelCase__       =       [c**2 for c in self.__components]
										return math.sqrt(sum(_lowerCAmelCase	)	)






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase = False	):
										lowerCamelCase__       =       self * other
										lowerCamelCase__       =       self.euclidean_length() * other.euclidean_length()
										if deg:
															return math.degrees(math.acos(num / den	)	)
										else:
															return math.acos(num / den	)





def    __UpperCamelCase      (    a) ->Vector:
					assert isinstance(a,     a)
					return Vector([0] * dimension)





def    __UpperCamelCase      (    a,     a) ->Vector:
					assert isinstance(a,     a) and (isinstance(a,     a))
					lowerCamelCase__       =       [0] * dimension
					lowerCamelCase__       =       1
					return Vector(a)





def    __UpperCamelCase      (    a,     a,     a) ->Vector:
					assert (
					    isinstance(a,     a)
					    and isinstance(a,     a)
					    and (isinstance(a,     (int, float)))
					)
					return x * scalar + y





def    __UpperCamelCase      (    a,     a,     a) ->Vector:
					random.seed(a)
					lowerCamelCase__       =       [random.randint(a,     a) for _ in range(a)]
					return Vector(a)

class SCREAMING_SNAKE_CASE_ :





					"""simple docstring"""


					def __init__( self	,  _lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										lowerCamelCase__       =       matrix
										lowerCamelCase__       =       w
										lowerCamelCase__       =       h





					def __str__( self	):
										lowerCamelCase__       =       ""
										for i in range(self.__height	):
															ans += "|"
															for j in range(self.__width	):
																				if j < self.__width - 1:
																									ans += str(self.__matrix[i][j]	) + ","
																				else:
																									ans += str(self.__matrix[i][j]	) + "|\n"
										return ans





					def __add__( self	,  _lowerCAmelCase	):
										if self.__width == other.width() and self.__height == other.height():
															lowerCamelCase__       =       []
															for i in range(self.__height	):
																				lowerCamelCase__       =       [
																				    self.__matrix[i][j] + other.component(_lowerCAmelCase	,  _lowerCAmelCase	)
																				    for j in range(self.__width	)
																				]
																				matrix.append(_lowerCAmelCase	)
															return Matrix(_lowerCAmelCase	,  self.__width	,  self.__height	)
										else:
															raise Exception("matrix must have the same dimension!"	)





					def __sub__( self	,  _lowerCAmelCase	):
										if self.__width == other.width() and self.__height == other.height():
															lowerCamelCase__       =       []
															for i in range(self.__height	):
																				lowerCamelCase__       =       [
																				    self.__matrix[i][j] - other.component(_lowerCAmelCase	,  _lowerCAmelCase	)
																				    for j in range(self.__width	)
																				]
																				matrix.append(_lowerCAmelCase	)
															return Matrix(_lowerCAmelCase	,  self.__width	,  self.__height	)
										else:
															raise Exception("matrices must have the same dimension!"	)





					@overload
					def __mul__( self	,  _lowerCAmelCase	):
										...





					@overload
					def __mul__( self	,  _lowerCAmelCase	):
										...





					def __mul__( self	,  _lowerCAmelCase	):
										if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	):  # matrix-vector
															if len(_lowerCAmelCase	) == self.__width:
																				lowerCamelCase__       =       zero_vector(self.__height	)
																				for i in range(self.__height	):
																									lowerCamelCase__       =       [
																									    self.__matrix[i][j] * other.component(_lowerCAmelCase	)
																									    for j in range(self.__width	)
																									]
																									ans.change_component(_lowerCAmelCase	,  sum(_lowerCAmelCase	)	)
																				return ans
															else:
																				raise Exception(
																				    "vector must have the same size as the "
																				    "number of columns of the matrix!"	)
										elif isinstance(_lowerCAmelCase	,  (int, float)	):  # matrix-scalar
															lowerCamelCase__       =       [
															    [self.__matrix[i][j] * other for j in range(self.__width	)]
															    for i in range(self.__height	)
															]
															return Matrix(_lowerCAmelCase	,  self.__width	,  self.__height	)
										return None





					def 					__magic_name__ ( self	):
										return self.__height





					def 					__magic_name__ ( self	):
										return self.__width





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										if 0 <= x < self.__height and 0 <= y < self.__width:
															return self.__matrix[x][y]
										else:
															raise Exception("change_component: indices out of bounds"	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										if 0 <= x < self.__height and 0 <= y < self.__width:
															lowerCamelCase__       =       value
										else:
															raise Exception("change_component: indices out of bounds"	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										if self.__height != self.__width:
															raise Exception("Matrix is not square"	)
										lowerCamelCase__       =       self.__matrix[:x] + self.__matrix[x + 1 :]
										for i in range(len(_lowerCAmelCase	)	):
															lowerCamelCase__       =       minor[i][:y] + minor[i][y + 1 :]
										return Matrix(_lowerCAmelCase	,  self.__width - 1	,  self.__height - 1	).determinant()





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										if self.__height != self.__width:
															raise Exception("Matrix is not square"	)
										if 0 <= x < self.__height and 0 <= y < self.__width:
															return (-1) ** (x + y) * self.minor(_lowerCAmelCase	,  _lowerCAmelCase	)
										else:
															raise Exception("Indices out of bounds"	)






					def 					__magic_name__ ( self	):
										if self.__height != self.__width:
															raise Exception("Matrix is not square"	)
										if self.__height < 1:
															raise Exception("Matrix has no element"	)
										elif self.__height == 1:
															return self.__matrix[0][0]
										elif self.__height == 2:
															return (
															    self.__matrix[0][0] * self.__matrix[1][1]
															    - self.__matrix[0][1] * self.__matrix[1][0]
															)
										else:
															lowerCamelCase__       =       [
															    self.__matrix[0][y] * self.cofactor(0	,  _lowerCAmelCase	) for y in range(self.__width	)
															]
															return sum(_lowerCAmelCase	)





def    __UpperCamelCase      (    a) ->Matrix:
					lowerCamelCase__       =       [[0] * n for _ in range(a)]
					return Matrix(a,     a,     a)







def    __UpperCamelCase      (    a,     a,     a,     a) ->Matrix:
					random.seed(a)
					lowerCamelCase__       =       [
					    [random.randint(a,     a) for _ in range(a)] for _ in range(a)
					]
					return Matrix(a,     a,     a)

