

"""simple docstring"""


import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE_    ):



    '''simple docstring'''




    def UpperCamelCase__		( self    )    ->      List[Any]:
          A				  =		tempfile.mkdtemp()
          A				  =		5

          # Realm tok
          A				  =		[
              '[UNK]',
              '[CLS]',
              '[SEP]',
              '[PAD]',
              '[MASK]',
              'test',
              'question',
              'this',
              'is',
              'the',
              'first',
              'second',
              'third',
              'fourth',
              'fifth',
              'record',
              'want',
              '##want',
              '##ed',
              'wa',
              'un',
              'runn',
              '##ing',
              ',',
              'low',
              'lowest',
          ]
          A				  =		os.path.join(self.tmpdirname ,"""realm_tokenizer"""    )
          os.makedirs(snake_case__ ,exist_ok=snake_case__    )
          A				  =		os.path.join(snake_case__ ,VOCAB_FILES_NAMES["""vocab_file"""]    )
          with open(self.vocab_file ,"""w""" ,encoding="""utf-8"""    ) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )

          A				  =		os.path.join(self.tmpdirname ,"""realm_block_records"""    )
          os.makedirs(snake_case__ ,exist_ok=snake_case__    )




    def UpperCamelCase__		( self    )    ->      Optional[Any]:
          return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname ,"""realm_tokenizer"""    )    )




    def UpperCamelCase__		( self    )    ->      Dict:
          shutil.rmtree(self.tmpdirname    )




    def UpperCamelCase__		( self    )    ->      List[str]:
          A				  =		RealmConfig(num_block_records=self.num_block_records    )
          return config




    def UpperCamelCase__		( self    )    ->      Dict:
          A				  =		Dataset.from_dict(
              {
                  """id""": ["""0""", """1"""],
                  """question""": ["""foo""", """bar"""],
                  """answers""": [["""Foo""", """Bar"""], ["""Bar"""]],
              }    )
          return dataset




    def UpperCamelCase__		( self    )    ->      Any:
          A				  =		np.array(
              [
                  b"""This is the first record""",
                  b"""This is the second record""",
                  b"""This is the third record""",
                  b"""This is the fourth record""",
                  b"""This is the fifth record""",
                  b"""This is a longer longer longer record""",
              ] ,dtype=snake_case__ ,)
          return block_records




    def UpperCamelCase__		( self    )    ->      Union[str, Any]:
          A				  =		RealmRetriever(
              block_records=self.get_dummy_block_records() ,tokenizer=self.get_tokenizer() ,)
          return retriever




    def UpperCamelCase__		( self    )    ->      List[str]:
          A				  =		self.get_config()
          A				  =		self.get_dummy_retriever()
          A				  =		retriever.tokenizer

          A				  =		np.array([0, 3] ,dtype="""long"""    )
          A				  =		tokenizer(["""Test question"""]    ).input_ids
          A				  =		tokenizer(
              ["""the fourth"""] ,add_special_tokens=snake_case__ ,return_token_type_ids=snake_case__ ,return_attention_mask=snake_case__ ,).input_ids
          A				  =		config.reader_seq_len

          A				  =		retriever(
              snake_case__ ,snake_case__ ,answer_ids=snake_case__ ,max_length=snake_case__ ,return_tensors="""np"""    )

          self.assertEqual(len(snake_case__    ) ,2    )
          self.assertEqual(len(snake_case__    ) ,2    )
          self.assertEqual(len(snake_case__    ) ,2    )
          self.assertEqual(concat_inputs.input_ids.shape ,(2, 1_0)    )
          self.assertEqual(concat_inputs.attention_mask.shape ,(2, 1_0)    )
          self.assertEqual(concat_inputs.token_type_ids.shape ,(2, 1_0)    )
          self.assertEqual(concat_inputs.special_tokens_mask.shape ,(2, 1_0)    )
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]    ) ,["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """first""", """record""", """[SEP]"""] ,)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]    ) ,["""[CLS]""", """test""", """question""", """[SEP]""", """this""", """is""", """the""", """fourth""", """record""", """[SEP]"""] ,)




    def UpperCamelCase__		( self    )    ->      Optional[int]:
          A				  =		self.get_config()
          A				  =		self.get_dummy_retriever()
          A				  =		retriever.tokenizer

          A				  =		np.array([0, 3, 5] ,dtype="""long"""    )
          A				  =		tokenizer(["""Test question"""]    ).input_ids
          A				  =		tokenizer(
              ["""the fourth""", """longer longer"""] ,add_special_tokens=snake_case__ ,return_token_type_ids=snake_case__ ,return_attention_mask=snake_case__ ,).input_ids
          A				  =		config.reader_seq_len

          A				  =		retriever(
              snake_case__ ,snake_case__ ,answer_ids=snake_case__ ,max_length=snake_case__ ,return_tensors="""np"""    )

          self.assertEqual([False, True, True] ,snake_case__    )
          self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]] ,snake_case__    )
          self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]] ,snake_case__    )




    def UpperCamelCase__		( self    )    ->      Union[str, Any]:
          A				  =		self.get_dummy_retriever()
          retriever.save_pretrained(os.path.join(self.tmpdirname ,"""realm_block_records"""    )    )

          # Test local path
          A				  =		retriever.from_pretrained(os.path.join(self.tmpdirname ,"""realm_block_records"""    )    )
          self.assertEqual(retriever.block_records[0] ,b"""This is the first record"""    )

          # Test mocked remote path
          with patch("""transformers.models.realm.retrieval_realm.hf_hub_download"""    ) as mock_hf_hub_download:
                A				  =		os.path.join(
                    os.path.join(self.tmpdirname ,"""realm_block_records"""    ) ,_REALM_BLOCK_RECORDS_FILENAME    )
                A				  =		RealmRetriever.from_pretrained("""google/realm-cc-news-pretrained-openqa"""    )

          self.assertEqual(retriever.block_records[0] ,b"""This is the first record"""    )








'''simple docstring'''







from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand






def       lowercase					(_A       ):







       """simple docstring"""







       return ConvertCommand(
           args.model_type ,    args.tf_checkpoint ,    args.pytorch_dump_output ,    args.config ,    args.finetuning_task_name       )


lowerCAmelCase     :  Optional[int]											=    """
transformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires
TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.
"""
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_						):



     """simple docstring"""






     @staticmethod
     def a						(			snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							parser.add_parser(
                'convert'   ,						help='CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.'   ,						)
            train_parser.add_argument('--model_type'   ,						type=snake_case__   ,						required=snake_case__   ,						help='Model\'s type.' )
            train_parser.add_argument(
                '--tf_checkpoint'   ,						type=snake_case__   ,						required=snake_case__   ,						help='TensorFlow checkpoint path or folder.' )
            train_parser.add_argument(
                '--pytorch_dump_output'   ,						type=snake_case__   ,						required=snake_case__   ,						help='Path to the PyTorch saved model output.' )
            train_parser.add_argument('--config'   ,						type=snake_case__   ,						default=''   ,						help='Configuration file path or folder.' )
            train_parser.add_argument(
                '--finetuning_task_name'   ,						type=snake_case__   ,						default=snake_case__   ,						help='Optional fine-tuning task name if the TF model was a finetuned model.'   ,						)
            train_parser.set_defaults(func=snake_case__ )







     def __init__(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						*snake_case__   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							logging.get_logger('transformers-cli/converting' )

            self._logger.info(F'Loading model {model_type}' )
            _lowerCAmelCase     :				Any       =							model_type
            _lowerCAmelCase     :				Any       =							tf_checkpoint
            _lowerCAmelCase     :				List[str]       =							pytorch_dump_output
            _lowerCAmelCase     :				int       =							config
            _lowerCAmelCase     :				str       =							finetuning_task_name







     def a						(			self ):

            '''simple docstring'''

            if self._model_type == "albert":
                   try:
                          from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                              convert_tf_checkpoint_to_pytorch,
                          )
                   except ImportError:
                          raise ImportError(snake_case__ )

                   convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            elif self._model_type == "bert":
                   try:
                          from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                              convert_tf_checkpoint_to_pytorch,
                          )
                   except ImportError:
                          raise ImportError(snake_case__ )

                   convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            elif self._model_type == "funnel":
                   try:
                          from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                              convert_tf_checkpoint_to_pytorch,
                          )
                   except ImportError:
                          raise ImportError(snake_case__ )

                   convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            elif self._model_type == "t5":
                   try:
                          from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
                   except ImportError:
                          raise ImportError(snake_case__ )

                   convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            elif self._model_type == "gpt":
                   from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
                       convert_openai_checkpoint_to_pytorch,
                   )

                   convert_openai_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            elif self._model_type == "transfo_xl":
                   try:
                          from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                              convert_transfo_xl_checkpoint_to_pytorch,
                          )
                   except ImportError:
                          raise ImportError(snake_case__ )

                   if "ckpt" in self._tf_checkpoint.lower():
                          _lowerCAmelCase     :				List[Any]       =							self._tf_checkpoint
                          _lowerCAmelCase     :				Optional[int]       =							''
                   else:
                          _lowerCAmelCase     :				int       =							self._tf_checkpoint
                          _lowerCAmelCase     :				Dict       =							''
                   convert_transfo_xl_checkpoint_to_pytorch(
                       snake_case__   ,						self._config   ,						self._pytorch_dump_output   ,						snake_case__ )
            elif self._model_type == "gpt2":
                   try:
                          from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
                              convert_gpta_checkpoint_to_pytorch,
                          )
                   except ImportError:
                          raise ImportError(snake_case__ )

                   convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            elif self._model_type == "xlnet":
                   try:
                          from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                              convert_xlnet_checkpoint_to_pytorch,
                          )
                   except ImportError:
                          raise ImportError(snake_case__ )

                   convert_xlnet_checkpoint_to_pytorch(
                       self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output   ,						self._finetuning_task_name )
            elif self._model_type == "xlm":
                   from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
                       convert_xlm_checkpoint_to_pytorch,
                   )

                   convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._pytorch_dump_output )
            elif self._model_type == "lxmert":
                   from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
                       convert_lxmert_checkpoint_to_pytorch,
                   )

                   convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._pytorch_dump_output )
            elif self._model_type == "rembert":
                   from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
                       convert_rembert_tf_checkpoint_to_pytorch,
                   )

                   convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,						self._config   ,						self._pytorch_dump_output )
            else:
                   raise ValueError(
                       '--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]' )

