



'''simple docstring'''





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer







class        UpperCAmelCase_			(			_a      ):
    """simple docstring"""







    def lowerCamelCase			(       self   :	Dict		):
          snake_case__    :       Union[str, Any]       		= tempfile.mkdtemp()
          snake_case__    :       Dict       		= 5

          # Realm tok
          snake_case__    :       Tuple       		= [
              """[UNK]""",
              """[CLS]""",
              """[SEP]""",
              """[PAD]""",
              """[MASK]""",
              """test""",
              """question""",
              """this""",
              """is""",
              """the""",
              """first""",
              """second""",
              """third""",
              """fourth""",
              """fifth""",
              """record""",
              """want""",
              """##want""",
              """##ed""",
              """wa""",
              """un""",
              """runn""",
              """##ing""",
              """,""",
              """low""",
              """lowest""",
          ]
          snake_case__    :       Optional[int]       		= os.path.join(self.tmpdirname					,	"""realm_tokenizer"""		)
          os.makedirs(snake_case_					,	exist_ok=snake_case_		)
          snake_case__    :       Union[str, Any]       		= os.path.join(snake_case_					,	VOCAB_FILES_NAMES["""vocab_file"""]		)
          with open(self.vocab_file					,	"""w"""					,	encoding="""utf-8"""		) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

          snake_case__    :       Dict       		= os.path.join(self.tmpdirname					,	"""realm_block_records"""		)
          os.makedirs(snake_case_					,	exist_ok=snake_case_		)




    def lowerCamelCase			(       self   :	str		):
          return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,	"""realm_tokenizer"""		)		)




    def lowerCamelCase			(       self   :	Dict		):
          shutil.rmtree(self.tmpdirname		)




    def lowerCamelCase			(       self   :	str		):
          snake_case__    :       List[str]       		= RealmConfig(num_block_records=self.num_block_records		)
          return config




    def lowerCamelCase			(       self   :	Union[str, Any]		):
          snake_case__    :       Any       		= Dataset.from_dict(
              {
                  """id""": ["""0""", """1"""],
                  """question""": ["""foo""", """bar"""],
                  """answers""": [["""Foo""", """Bar"""], ["""Bar"""]],
              }		)
          return dataset




    def lowerCamelCase			(       self   :	List[Any]		):
          snake_case__    :       Any       		= np.array(
              [
                  b"""This is the first record""",
                  b"""This is the second record""",
                  b"""This is the third record""",
                  b"""This is the fourth record""",
                  b"""This is the fifth record""",
                  b"""This is a longer longer longer record""",
              ]					,	dtype=snake_case_					,	)
          return block_records




    def lowerCamelCase			(       self   :	Dict		):
          snake_case__    :       Any       		= RealmRetriever(
              block_records=self.get_dummy_block_records()					,	tokenizer=self.get_tokenizer()					,	)
          return retriever




    def lowerCamelCase			(       self   :	int		):
          snake_case__    :       Tuple       		= self.get_config()
          snake_case__    :       Tuple       		= self.get_dummy_retriever()
          snake_case__    :       int       		= retriever.tokenizer

          snake_case__    :       int       		= np.array([0, 3]					,	dtype="""long"""		)
          snake_case__    :       int       		= tokenizer(["""Test question"""]		).input_ids
          snake_case__    :       Union[str, Any]       		= tokenizer(
              ["""the fourth"""]					,	add_special_tokens=snake_case_					,	return_token_type_ids=snake_case_					,	return_attention_mask=snake_case_					,	).input_ids
          snake_case__    :       Optional[int]       		= config.reader_seq_len

          snake_case__		,							snake_case__		,							snake_case__		,							snake_case__    :       int       		= retriever(
              snake_case_					,	snake_case_					,	answer_ids=snake_case_					,	max_length=snake_case_					,	return_tensors="""np"""		)

          self.assertEqual(len(snake_case_		)					,	2		)
          self.assertEqual(len(snake_case_		)					,	2		)
          self.assertEqual(len(snake_case_		)					,	2		)
          self.assertEqual(concat_inputs.input_ids.shape					,	(2, 10)		)
          self.assertEqual(concat_inputs.attention_mask.shape					,	(2, 10)		)
          self.assertEqual(concat_inputs.token_type_ids.shape					,	(2, 10)		)
          self.assertEqual(concat_inputs.special_tokens_mask.shape					,	(2, 10)		)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]		)					,	["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """first""", """record""", """[SEP]"""]					,	)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]		)					,	["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """fourth""", """record""", """[SEP]"""]					,	)




    def lowerCamelCase			(       self   :	Dict		):
          snake_case__    :       Tuple       		= self.get_config()
          snake_case__    :       str       		= self.get_dummy_retriever()
          snake_case__    :       Tuple       		= retriever.tokenizer

          snake_case__    :       List[str]       		= np.array([0, 3, 5]					,	dtype="""long"""		)
          snake_case__    :       Optional[Any]       		= tokenizer(["""Test question"""]		).input_ids
          snake_case__    :       List[Any]       		= tokenizer(
              ["""the fourth""", """longer longer"""]					,	add_special_tokens=snake_case_					,	return_token_type_ids=snake_case_					,	return_attention_mask=snake_case_					,	).input_ids
          snake_case__    :       Optional[Any]       		= config.reader_seq_len

          snake_case__		,							snake_case__		,							snake_case__		,							snake_case__    :       Tuple       		= retriever(
              snake_case_					,	snake_case_					,	answer_ids=snake_case_					,	max_length=snake_case_					,	return_tensors="""np"""		)

          self.assertEqual([False, True, True]					,	snake_case_		)
          self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,	snake_case_		)
          self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,	snake_case_		)




    def lowerCamelCase			(       self   :	int		):
          snake_case__    :       Tuple       		= self.get_dummy_retriever()
          retriever.save_pretrained(os.path.join(self.tmpdirname					,	"""realm_block_records"""		)		)

          # Test local path
          snake_case__    :       Any       		= retriever.from_pretrained(os.path.join(self.tmpdirname					,	"""realm_block_records"""		)		)
          self.assertEqual(retriever.block_records[0]					,	b"""This is the first record"""		)

          # Test mocked remote path
          with patch("""transformers.models.realm.retrieval_realm.hf_hub_download"""		) as mock_hf_hub_download:
                snake_case__    :       Union[str, Any]       		= os.path.join(
                    os.path.join(self.tmpdirname					,	"""realm_block_records"""		)					,	_REALM_BLOCK_RECORDS_FILENAME		)
                snake_case__    :       Any       		= RealmRetriever.from_pretrained("""google/realm-cc-news-pretrained-openqa"""		)

          self.assertEqual(retriever.block_records[0]					,	b"""This is the first record"""		)





'''simple docstring'''





# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


__a  			=						"3"

print("Python version:", sys.version)
print("transformers version:", transformers.__version__)

try:
       import torch

       print("Torch version:", torch.__version__)
       print("Cuda available:", torch.cuda.is_available())
       print("Cuda version:", torch.version.cuda)
       print("CuDNN version:", torch.backends.cudnn.version())
       print("Number of GPUs available:", torch.cuda.device_count())
       print("NCCL version:", torch.cuda.nccl.version())
except ImportError:
       print("Torch version:", None)

try:
       import deepspeed

       print("DeepSpeed version:", deepspeed.__version__)
except ImportError:
       print("DeepSpeed version:", None)

try:
       import tensorflow as tf

       print("TensorFlow version:", tf.__version__)
       print("TF GPUs available:", bool(tf.config.list_physical_devices("GPU")))
       print("Number of TF GPUs available:", len(tf.config.list_physical_devices("GPU")))
except ImportError:
       print("TensorFlow version:", None)
