

"""simple docstring"""






import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer
class        _lowercase					(     __UpperCAmelCase     ):







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      tempfile.mkdtemp()
      __magic_name__     =      5

      # Realm tok
      __magic_name__     =      [
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''test''',
          '''question''',
          '''this''',
          '''is''',
          '''the''',
          '''first''',
          '''second''',
          '''third''',
          '''fourth''',
          '''fifth''',
          '''record''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      __magic_name__     =      os.path.join(self.tmpdirname ,						'''realm_tokenizer'''       )
      os.makedirs(UpperCamelCase_ ,						exist_ok=UpperCamelCase_       )
      __magic_name__     =      os.path.join(UpperCamelCase_ ,						VOCAB_FILES_NAMES['''vocab_file''']       )
      with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )

      __magic_name__     =      os.path.join(self.tmpdirname ,						'''realm_block_records'''       )
      os.makedirs(UpperCamelCase_ ,						exist_ok=UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname ,						'''realm_tokenizer'''       )       )







     def 	lowerCAmelCase__    (      self       ):
      shutil.rmtree(self.tmpdirname       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      RealmConfig(num_block_records=self.num_block_records       )
      return config







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Dataset.from_dict(
          {
              '''id''': ['''0''', '''1'''],
              '''question''': ['''foo''', '''bar'''],
              '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
          }       )
      return dataset







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      np.array(
          [
              B'''This is the first record''',
              B'''This is the second record''',
              B'''This is the third record''',
              B'''This is the fourth record''',
              B'''This is the fifth record''',
              B'''This is a longer longer longer record''',
          ] ,						dtype=UpperCamelCase_ ,						)
      return block_records







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      RealmRetriever(
          block_records=self.get_dummy_block_records() ,						tokenizer=self.get_tokenizer() ,						)
      return retriever







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_config()
      __magic_name__     =      self.get_dummy_retriever()
      __magic_name__     =      retriever.tokenizer

      __magic_name__     =      np.array([0, 3] ,						dtype='''long'''       )
      __magic_name__     =      tokenizer(['''Test question''']       ).input_ids
      __magic_name__     =      tokenizer(
          ['''the fourth'''] ,						add_special_tokens=UpperCamelCase_ ,						return_token_type_ids=UpperCamelCase_ ,						return_attention_mask=UpperCamelCase_ ,						).input_ids
      __magic_name__     =      config.reader_seq_len

      __magic_name__   ,						__magic_name__   ,						__magic_name__   ,						__magic_name__     =      retriever(
          UpperCamelCase_ ,						UpperCamelCase_ ,						answer_ids=UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						return_tensors='''np'''       )

      self.assertEqual(len(UpperCamelCase_       ) ,						2       )
      self.assertEqual(len(UpperCamelCase_       ) ,						2       )
      self.assertEqual(len(UpperCamelCase_       ) ,						2       )
      self.assertEqual(concat_inputs.input_ids.shape ,						(2, 10)       )
      self.assertEqual(concat_inputs.attention_mask.shape ,						(2, 10)       )
      self.assertEqual(concat_inputs.token_type_ids.shape ,						(2, 10)       )
      self.assertEqual(concat_inputs.special_tokens_mask.shape ,						(2, 10)       )
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]       ) ,						['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]'''] ,						)
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]       ) ,						['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]'''] ,						)







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_config()
      __magic_name__     =      self.get_dummy_retriever()
      __magic_name__     =      retriever.tokenizer

      __magic_name__     =      np.array([0, 3, 5] ,						dtype='''long'''       )
      __magic_name__     =      tokenizer(['''Test question''']       ).input_ids
      __magic_name__     =      tokenizer(
          ['''the fourth''', '''longer longer'''] ,						add_special_tokens=UpperCamelCase_ ,						return_token_type_ids=UpperCamelCase_ ,						return_attention_mask=UpperCamelCase_ ,						).input_ids
      __magic_name__     =      config.reader_seq_len

      __magic_name__   ,						__magic_name__   ,						__magic_name__   ,						__magic_name__     =      retriever(
          UpperCamelCase_ ,						UpperCamelCase_ ,						answer_ids=UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						return_tensors='''np'''       )

      self.assertEqual([False, True, True] ,						UpperCamelCase_       )
      self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]] ,						UpperCamelCase_       )
      self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]] ,						UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_dummy_retriever()
      retriever.save_pretrained(os.path.join(self.tmpdirname ,						'''realm_block_records'''       )       )

      # Test local path
      __magic_name__     =      retriever.from_pretrained(os.path.join(self.tmpdirname ,						'''realm_block_records'''       )       )
      self.assertEqual(retriever.block_records[0] ,						B'''This is the first record'''       )

      # Test mocked remote path
      with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''       ) as mock_hf_hub_download:
       __magic_name__     =      os.path.join(
           os.path.join(self.tmpdirname ,						'''realm_block_records'''       ) ,						_REALM_BLOCK_RECORDS_FILENAME       )
       __magic_name__     =      RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''       )

      self.assertEqual(retriever.block_records[0] ,						B'''This is the first record'''       )




"""simple docstring"""






from typing import List, Optional, Union

import numpy as np
import PIL.Image

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__lowerCamelCase										=		logging.get_logger(__name__)
class        _lowercase					(     __UpperCAmelCase     ):
     _lowerCamelCase     =       ['''pixel_values''']







     def __init__(      self ,						UpperCamelCase_ = True ,						UpperCamelCase_ = 32 ,						UpperCamelCase_=PILImageResampling.BILINEAR ,						UpperCamelCase_ = True ,						**UpperCamelCase_ ,						):
      __magic_name__     =      do_resize
      __magic_name__     =      do_rescale
      __magic_name__     =      size_divisor
      __magic_name__     =      resample
      super().__init__(**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_ = None ,						**UpperCamelCase_       ):
      __magic_name__   ,						__magic_name__     =      get_image_size(UpperCamelCase_       )
      # Rounds the height and width down to the closest multiple of size_divisor
      __magic_name__     =      height // size_divisor * size_divisor
      __magic_name__     =      width // size_divisor * size_divisor
      __magic_name__     =      resize(UpperCamelCase_ ,						(new_h, new_w) ,						resample=UpperCamelCase_ ,						data_format=UpperCamelCase_ ,						**UpperCamelCase_       )
      return image







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_ = None ,						**UpperCamelCase_       ):
      return rescale(image=UpperCamelCase_ ,						scale=UpperCamelCase_ ,						data_format=UpperCamelCase_ ,						**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None ,						UpperCamelCase_ = None ,						UpperCamelCase_=None ,						UpperCamelCase_ = None ,						UpperCamelCase_ = None ,						UpperCamelCase_ = ChannelDimension.FIRST ,						**UpperCamelCase_ ,						):
      __magic_name__     =      do_resize if do_resize is not None else self.do_resize
      __magic_name__     =      do_rescale if do_rescale is not None else self.do_rescale
      __magic_name__     =      size_divisor if size_divisor is not None else self.size_divisor
      __magic_name__     =      resample if resample is not None else self.resample

      if do_resize and size_divisor is None:
       raise ValueError('''size_divisor is required for resizing'''       )

      __magic_name__     =      make_list_of_images(UpperCamelCase_       )

      if not valid_images(UpperCamelCase_       ):
       raise ValueError('''Invalid image(s)'''       )

      # All transformations expect numpy arrays.
      __magic_name__     =      [to_numpy_array(UpperCamelCase_       ) for img in images]

      if do_resize:
       __magic_name__     =      [self.resize(UpperCamelCase_ ,						size_divisor=UpperCamelCase_ ,						resample=UpperCamelCase_       ) for image in images]

      if do_rescale:
       __magic_name__     =      [self.rescale(UpperCamelCase_ ,						scale=1 / 255       ) for image in images]

      __magic_name__     =      [to_channel_dimension_format(UpperCamelCase_ ,						UpperCamelCase_       ) for image in images]

      __magic_name__     =      {'''pixel_values''': images}
      return BatchFeature(data=UpperCamelCase_ ,						tensor_type=UpperCamelCase_       )

