

"""simple docstring"""






import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer
class        _lowercase					(     __UpperCAmelCase     ):







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      tempfile.mkdtemp()
      __magic_name__     =      5

      # Realm tok
      __magic_name__     =      [
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''test''',
          '''question''',
          '''this''',
          '''is''',
          '''the''',
          '''first''',
          '''second''',
          '''third''',
          '''fourth''',
          '''fifth''',
          '''record''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      __magic_name__     =      os.path.join(self.tmpdirname ,						'''realm_tokenizer'''       )
      os.makedirs(UpperCamelCase_ ,						exist_ok=UpperCamelCase_       )
      __magic_name__     =      os.path.join(UpperCamelCase_ ,						VOCAB_FILES_NAMES['''vocab_file''']       )
      with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )

      __magic_name__     =      os.path.join(self.tmpdirname ,						'''realm_block_records'''       )
      os.makedirs(UpperCamelCase_ ,						exist_ok=UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname ,						'''realm_tokenizer'''       )       )







     def 	lowerCAmelCase__    (      self       ):
      shutil.rmtree(self.tmpdirname       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      RealmConfig(num_block_records=self.num_block_records       )
      return config







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Dataset.from_dict(
          {
              '''id''': ['''0''', '''1'''],
              '''question''': ['''foo''', '''bar'''],
              '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
          }       )
      return dataset







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      np.array(
          [
              B'''This is the first record''',
              B'''This is the second record''',
              B'''This is the third record''',
              B'''This is the fourth record''',
              B'''This is the fifth record''',
              B'''This is a longer longer longer record''',
          ] ,						dtype=UpperCamelCase_ ,						)
      return block_records







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      RealmRetriever(
          block_records=self.get_dummy_block_records() ,						tokenizer=self.get_tokenizer() ,						)
      return retriever







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_config()
      __magic_name__     =      self.get_dummy_retriever()
      __magic_name__     =      retriever.tokenizer

      __magic_name__     =      np.array([0, 3] ,						dtype='''long'''       )
      __magic_name__     =      tokenizer(['''Test question''']       ).input_ids
      __magic_name__     =      tokenizer(
          ['''the fourth'''] ,						add_special_tokens=UpperCamelCase_ ,						return_token_type_ids=UpperCamelCase_ ,						return_attention_mask=UpperCamelCase_ ,						).input_ids
      __magic_name__     =      config.reader_seq_len

      __magic_name__   ,						__magic_name__   ,						__magic_name__   ,						__magic_name__     =      retriever(
          UpperCamelCase_ ,						UpperCamelCase_ ,						answer_ids=UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						return_tensors='''np'''       )

      self.assertEqual(len(UpperCamelCase_       ) ,						2       )
      self.assertEqual(len(UpperCamelCase_       ) ,						2       )
      self.assertEqual(len(UpperCamelCase_       ) ,						2       )
      self.assertEqual(concat_inputs.input_ids.shape ,						(2, 10)       )
      self.assertEqual(concat_inputs.attention_mask.shape ,						(2, 10)       )
      self.assertEqual(concat_inputs.token_type_ids.shape ,						(2, 10)       )
      self.assertEqual(concat_inputs.special_tokens_mask.shape ,						(2, 10)       )
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]       ) ,						['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]'''] ,						)
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]       ) ,						['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]'''] ,						)







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_config()
      __magic_name__     =      self.get_dummy_retriever()
      __magic_name__     =      retriever.tokenizer

      __magic_name__     =      np.array([0, 3, 5] ,						dtype='''long'''       )
      __magic_name__     =      tokenizer(['''Test question''']       ).input_ids
      __magic_name__     =      tokenizer(
          ['''the fourth''', '''longer longer'''] ,						add_special_tokens=UpperCamelCase_ ,						return_token_type_ids=UpperCamelCase_ ,						return_attention_mask=UpperCamelCase_ ,						).input_ids
      __magic_name__     =      config.reader_seq_len

      __magic_name__   ,						__magic_name__   ,						__magic_name__   ,						__magic_name__     =      retriever(
          UpperCamelCase_ ,						UpperCamelCase_ ,						answer_ids=UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						return_tensors='''np'''       )

      self.assertEqual([False, True, True] ,						UpperCamelCase_       )
      self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]] ,						UpperCamelCase_       )
      self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]] ,						UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_dummy_retriever()
      retriever.save_pretrained(os.path.join(self.tmpdirname ,						'''realm_block_records'''       )       )

      # Test local path
      __magic_name__     =      retriever.from_pretrained(os.path.join(self.tmpdirname ,						'''realm_block_records'''       )       )
      self.assertEqual(retriever.block_records[0] ,						B'''This is the first record'''       )

      # Test mocked remote path
      with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''       ) as mock_hf_hub_download:
       __magic_name__     =      os.path.join(
           os.path.join(self.tmpdirname ,						'''realm_block_records'''       ) ,						_REALM_BLOCK_RECORDS_FILENAME       )
       __magic_name__     =      RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''       )

      self.assertEqual(retriever.block_records[0] ,						B'''This is the first record'''       )




"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCamelCase										=		{
    "configuration_m2m_100": ["M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP", "M2M100Config", "M2M100OnnxConfig"],
    "tokenization_m2m_100": ["M2M100Tokenizer"],
}


try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCamelCase										=		[
     "M2M_100_PRETRAINED_MODEL_ARCHIVE_LIST",
     "M2M100ForConditionalGeneration",
     "M2M100Model",
     "M2M100PreTrainedModel",
 ]


if TYPE_CHECKING:
 from .configuration_mam_aaa import M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP, MaMaaaConfig, MaMaaaOnnxConfig
 from .tokenization_mam_aaa import MaMaaaTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_mam_aaa import (
      M2M_100_PRETRAINED_MODEL_ARCHIVE_LIST,
      MaMaaaForConditionalGeneration,
      MaMaaaModel,
      MaMaaaPreTrainedModel,
  )


else:
 import sys

 __lowerCamelCase										=		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

