
'''simple docstring'''





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							tempfile.mkdtemp()
   UpperCAmelCase__      :					Any				       =							5

   # Realm tok
   UpperCAmelCase__      :					Union[str, Any]				       =							[
       "[UNK]",
       "[CLS]",
       "[SEP]",
       "[PAD]",
       "[MASK]",
       "test",
       "question",
       "this",
       "is",
       "the",
       "first",
       "second",
       "third",
       "fourth",
       "fifth",
       "record",
       "want",
       "##want",
       "##ed",
       "wa",
       "un",
       "runn",
       "##ing",
       ",",
       "low",
       "lowest",
   ]
   UpperCAmelCase__      :					List[str]				       =							os.path.join(self.tmpdirname			,				"realm_tokenizer" )
   os.makedirs(snake_case__			,				exist_ok=snake_case__ )
   UpperCAmelCase__      :					Optional[Any]				       =							os.path.join(snake_case__			,				VOCAB_FILES_NAMES["vocab_file"] )
   with open(self.vocab_file			,				"w"			,				encoding="utf-8" ) as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )

   UpperCAmelCase__      :					Optional[int]				       =							os.path.join(self.tmpdirname			,				"realm_block_records" )
   os.makedirs(snake_case__			,				exist_ok=snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname			,				"realm_tokenizer" ) )







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   shutil.rmtree(self.tmpdirname )







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[str]				       =							RealmConfig(num_block_records=self.num_block_records )
   return config







 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					int				       =							Dataset.from_dict(
       {
           "id": ["0", "1"],
           "question": ["foo", "bar"],
           "answers": [["Foo", "Bar"], ["Bar"]],
       } )
   return dataset







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							np.array(
       [
           B"This is the first record",
           B"This is the second record",
           B"This is the third record",
           B"This is the fourth record",
           B"This is the fifth record",
           B"This is a longer longer longer record",
       ]			,				dtype=snake_case__			,				)
   return block_records







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							RealmRetriever(
       block_records=self.get_dummy_block_records()			,				tokenizer=self.get_tokenizer()			,				)
   return retriever







 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							self.get_config()
   UpperCAmelCase__      :					int				       =							self.get_dummy_retriever()
   UpperCAmelCase__      :					Union[str, Any]				       =							retriever.tokenizer

   UpperCAmelCase__      :					Optional[Any]				       =							np.array([0, 3]			,				dtype="long" )
   UpperCAmelCase__      :					Dict				       =							tokenizer(["Test question"] ).input_ids
   UpperCAmelCase__      :					Optional[Any]				       =							tokenizer(
       ["the fourth"]			,				add_special_tokens=snake_case__			,				return_token_type_ids=snake_case__			,				return_attention_mask=snake_case__			,				).input_ids
   UpperCAmelCase__      :					List[Any]				       =							config.reader_seq_len

   UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					Any				       =							retriever(
       snake_case__			,				snake_case__			,				answer_ids=snake_case__			,				max_length=snake_case__			,				return_tensors="np" )

   self.assertEqual(len(snake_case__ )			,				2 )
   self.assertEqual(len(snake_case__ )			,				2 )
   self.assertEqual(len(snake_case__ )			,				2 )
   self.assertEqual(concat_inputs.input_ids.shape			,				(2, 10) )
   self.assertEqual(concat_inputs.attention_mask.shape			,				(2, 10) )
   self.assertEqual(concat_inputs.token_type_ids.shape			,				(2, 10) )
   self.assertEqual(concat_inputs.special_tokens_mask.shape			,				(2, 10) )
   self.assertEqual(
       tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0] )			,				["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "first", "record", "[SEP]"]			,				)
   self.assertEqual(
       tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1] )			,				["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "fourth", "record", "[SEP]"]			,				)







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							self.get_config()
   UpperCAmelCase__      :					List[str]				       =							self.get_dummy_retriever()
   UpperCAmelCase__      :					Optional[int]				       =							retriever.tokenizer

   UpperCAmelCase__      :					Optional[int]				       =							np.array([0, 3, 5]			,				dtype="long" )
   UpperCAmelCase__      :					Optional[int]				       =							tokenizer(["Test question"] ).input_ids
   UpperCAmelCase__      :					Union[str, Any]				       =							tokenizer(
       ["the fourth", "longer longer"]			,				add_special_tokens=snake_case__			,				return_token_type_ids=snake_case__			,				return_attention_mask=snake_case__			,				).input_ids
   UpperCAmelCase__      :					Any				       =							config.reader_seq_len

   UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					Dict				       =							retriever(
       snake_case__			,				snake_case__			,				answer_ids=snake_case__			,				max_length=snake_case__			,				return_tensors="np" )

   self.assertEqual([False, True, True]			,				snake_case__ )
   self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]			,				snake_case__ )
   self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							self.get_dummy_retriever()
   retriever.save_pretrained(os.path.join(self.tmpdirname			,				"realm_block_records" ) )

   # Test local path
   UpperCAmelCase__      :					List[str]				       =							retriever.from_pretrained(os.path.join(self.tmpdirname			,				"realm_block_records" ) )
   self.assertEqual(retriever.block_records[0]			,				B"This is the first record" )

   # Test mocked remote path
   with patch("transformers.models.realm.retrieval_realm.hf_hub_download" ) as mock_hf_hub_download:
     UpperCAmelCase__      :					Union[str, Any]				       =							os.path.join(
         os.path.join(self.tmpdirname			,				"realm_block_records" )			,				_REALM_BLOCK_RECORDS_FILENAME )
     UpperCAmelCase__      :					Any				       =							RealmRetriever.from_pretrained("google/realm-cc-news-pretrained-openqa" )

   self.assertEqual(retriever.block_records[0]			,				B"This is the first record" )





'''simple docstring'''





import math







def snake_case_     (					lowercase__		):

  if 1 < number < 4:
    # 2 and 3 are primes
    return True
  elif number < 2 or number % 2 == 0 or number % 3 == 0:
    # Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
    return False

  # All primes number are in format of 6k +/- 1
  for i in range(5   ,       int(math.sqrt(lowercase__		) + 1		)   ,       6		):
    if number % i == 0 or number % (i + 2) == 0:
      return False
  return True







def snake_case_     (					lowercase__ = 0.1		):
  UpperCAmelCase__      :					Dict				       =							3
  UpperCAmelCase__      :					Tuple				       =							3

  while primes / (2 * j - 1) >= ratio:
    for i in range(j * j + j + 1   ,       (j + 2) * (j + 2)   ,       j + 1		):
      primes += is_prime(lowercase__		)
    j += 2
  return j


if __name__ == "__main__":
      import doctest

      doctest.testmod()



