





import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch





def 					lowerCAmelCase_			(	__a		)					->		Dict:  # picklable for multiprocessing







  """simple docstring"""





  return x.sum()





def 					lowerCAmelCase_			(	__a		)					->		Optional[Any]:  # picklable for multiprocessing







  """simple docstring"""





  return i + 1



@dataclass
class        _SCREAMING_SNAKE_CASE						:







 '''simple docstring'''
 lowercase_		=      42
 lowercase_		=      42



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->Any:



   '''simple docstring'''



   lowerCamelCase__:				Optional[int]        ={}
   lowerCamelCase__:				Dict        =[]
   lowerCamelCase__:				Any        =1
   lowerCamelCase__:				List[str]        =[1, 2]
   lowerCamelCase__:				List[str]        ={"a": 1, "b": 2}
   lowerCamelCase__:				Optional[Any]        ={"a": [1, 2], "b": [3, 4]}
   lowerCamelCase__:				Union[str, Any]        ={"a": {"1": 1}, "b": 2}
   lowerCamelCase__:				Tuple        ={"a": 1, "b": 2, "c": 3, "d": 4}
   lowerCamelCase__:				Any        ={}
   lowerCamelCase__:				Union[str, Any]        =[]
   lowerCamelCase__:				int        =2
   lowerCamelCase__:				Dict        =[2, 3]
   lowerCamelCase__:				List[str]        ={"a": 2, "b": 3}
   lowerCamelCase__:				str        ={"a": [2, 3], "b": [4, 5]}
   lowerCamelCase__:				List[str]        ={"a": {"1": 2}, "b": 3}
   lowerCamelCase__:				Union[str, Any]        ={"a": 2, "b": 3, "c": 4, "d": 5}
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_)	,			UpperCAmelCase_)

   lowerCamelCase__:				List[str]        =2
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)

   lowerCamelCase__:				int        ={"a": np.eye(2), "b": np.zeros(3), "c": np.ones(2)}
   lowerCamelCase__:				Dict        ={"a": 2, "b": 0, "c": 2}
   lowerCamelCase__:				Dict        ={
       "a": np.eye(2).astype(UpperCAmelCase_),
       "b": np.zeros(3).astype(UpperCAmelCase_),
       "c": np.ones(2).astype(UpperCAmelCase_),
   }
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			map_numpy=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(
       {k: v.tolist() for k, v in map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			map_numpy=UpperCAmelCase_).items()}	,			{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}	,			)
   self.assertEqual(map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			map_numpy=UpperCAmelCase_	,			num_proc=UpperCAmelCase_)	,			UpperCAmelCase_)
   self.assertEqual(
       {k: v.tolist() for k, v in map_nested(UpperCAmelCase_	,			UpperCAmelCase_	,			map_numpy=UpperCAmelCase_	,			num_proc=UpperCAmelCase_).items()}	,			{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}	,			)
   with self.assertRaises(UpperCAmelCase_):  # can't pickle a local lambda
     map_nested(lambda UpperCAmelCase_: x + 1	,			UpperCAmelCase_	,			num_proc=UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple)	->int:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        ={"a": 1, "b": 2}
   lowerCamelCase__:				int        ={"a": 3, "b": 4}
   lowerCamelCase__:				Union[str, Any]        ={"a": 5, "b": 6}
   lowerCamelCase__:				List[Any]        =sorted([("a", (1, 3, 5)), ("b", (2, 4, 6))])
   self.assertEqual(sorted(zip_dict(UpperCAmelCase_	,			UpperCAmelCase_	,			UpperCAmelCase_))	,			UpperCAmelCase_)







 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str])	->str:



   '''simple docstring'''



   class        _SCREAMING_SNAKE_CASE						:







    '''simple docstring'''
    lowercase_		=      "bar"



   lowerCamelCase__:				Dict        =Foo()
   self.assertEqual(foo.my_attr	,			"bar")
   with temporary_assignment(UpperCAmelCase_	,			"my_attr"	,			"BAR"):
     self.assertEqual(foo.my_attr	,			"BAR")
   self.assertEqual(foo.my_attr	,			"bar")





@pytest.mark.parametrize(
    "iterable_length, num_proc, expected_num_proc"     ,   [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]     ,   )
def 					lowerCAmelCase_			(	__a     ,   __a     ,   __a		)					->		List[Any]:







  """simple docstring"""





  with patch("datasets.utils.py_utils._single_map_nested"		) as mock_single_map_nested, patch(
      "datasets.parallel.parallel.Pool"		) as mock_multiprocessing_pool:
    lowerCamelCase__:				List[str]        ={F"""{i}""": i for i in range(__a		)}
    lowerCamelCase__:				str        =map_nested(lambda __a		: x + 10     ,   __a     ,   num_proc=__a     ,   parallel_min_length=16		)
    if expected_num_proc == 1:
      assert mock_single_map_nested.called
      assert not mock_multiprocessing_pool.called
    else:
      assert not mock_single_map_nested.called
      assert mock_multiprocessing_pool.called
      assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''

 @require_tf
 def 							SCREAMING_SNAKE_CASE_ (self    :       Dict)	->int:



   '''simple docstring'''



   import tensorflow as tf
   from tensorflow.keras import layers

   lowerCamelCase__:				Optional[Any]        =layers.Dense(2)

   def gen_random_output():
     lowerCamelCase__:				Optional[int]        =tf.random.uniform((1, 3))
     return model(UpperCAmelCase_).numpy()

   with temp_seed(42	,			set_tensorflow=UpperCAmelCase_):
     lowerCamelCase__:				Optional[Any]        =gen_random_output()
   with temp_seed(42	,			set_tensorflow=UpperCAmelCase_):
     lowerCamelCase__:				Any        =gen_random_output()
   lowerCamelCase__:				Union[str, Any]        =gen_random_output()

   np.testing.assert_equal(UpperCAmelCase_	,			UpperCAmelCase_)
   self.assertGreater(np.abs(outa - outa).sum()	,			0)

 @require_torch
 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->str:



   '''simple docstring'''



   import torch

   def gen_random_output():
     lowerCamelCase__:				Union[str, Any]        =torch.nn.Linear(3	,			2)
     lowerCamelCase__:				List[Any]        =torch.rand(1	,			3)
     return model(UpperCAmelCase_).detach().numpy()

   with temp_seed(42	,			set_pytorch=UpperCAmelCase_):
     lowerCamelCase__:				Tuple        =gen_random_output()
   with temp_seed(42	,			set_pytorch=UpperCAmelCase_):
     lowerCamelCase__:				Tuple        =gen_random_output()
   lowerCamelCase__:				Optional[Any]        =gen_random_output()

   np.testing.assert_equal(UpperCAmelCase_	,			UpperCAmelCase_)
   self.assertGreater(np.abs(outa - outa).sum()	,			0)







 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->Union[str, Any]:



   '''simple docstring'''



   def gen_random_output():
     return np.random.rand(1	,			3)

   with temp_seed(42):
     lowerCamelCase__:				Optional[int]        =gen_random_output()
   with temp_seed(42):
     lowerCamelCase__:				Tuple        =gen_random_output()
   lowerCamelCase__:				Dict        =gen_random_output()

   np.testing.assert_equal(UpperCAmelCase_	,			UpperCAmelCase_)
   self.assertGreater(np.abs(outa - outa).sum()	,			0)





@pytest.mark.parametrize("input_data"     ,   [{}]		)
def 					lowerCAmelCase_			(	__a		)					->		Optional[Any]:







  """simple docstring"""





  lowerCamelCase__:				List[str]        =NestedDataStructure(__a		).data
  assert output_data == input_data





@pytest.mark.parametrize(
    "data, expected_output"     ,   [
        ({}, []),
        ([], []),
        ("foo", ["foo"]),
        (["foo", "bar"], ["foo", "bar"]),
        ([["foo", "bar"]], ["foo", "bar"]),
        ([[["foo"], ["bar"]]], ["foo", "bar"]),
        ([[["foo"], "bar"]], ["foo", "bar"]),
        ({"a": 1, "b": 2}, [1, 2]),
        ({"a": [1, 2], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[1, 2]], "b": [[3, 4]]}, [1, 2, 3, 4]),
        ({"a": [[1, 2]], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [[3, 4]]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [3, [4]]}, [1, 2, 3, 4]),
        ({"a": {"1": 1}, "b": 2}, [1, 2]),
        ({"a": {"1": [1]}, "b": 2}, [1, 2]),
        ({"a": {"1": [1]}, "b": [2]}, [1, 2]),
    ]     ,   )
def 					lowerCAmelCase_			(	__a     ,   __a		)					->		Optional[int]:







  """simple docstring"""





  lowerCamelCase__:				List[str]        =NestedDataStructure(__a		).flatten()
  assert output == expected_output





def 					lowerCAmelCase_			(	)					->		Optional[int]:







  """simple docstring"""





  lowerCamelCase__:				Optional[Any]        =A(x=1     ,   y="foobar"		)
  lowerCamelCase__:				Dict        ={"x": 1, "y": "foobar"}
  assert asdict(__a		) == expected_output

  lowerCamelCase__:				str        ={"a": {"b": A(x=10     ,   y="foo"		)}, "c": [A(x=20     ,   y="bar"		)]}
  lowerCamelCase__:				Optional[Any]        ={"a": {"b": {"x": 10, "y": "foo"}}, "c": [{"x": 20, "y": "bar"}]}
  assert asdict(__a		) == expected_output

  with pytest.raises(__a		):
    asdict([1, A(x=10     ,   y="foo"		)]		)





def 					lowerCAmelCase_			(	__a		)					->		List[Any]:







  """simple docstring"""





  return text.split()





def 					lowerCAmelCase_			(	__a		)					->		Optional[Any]:







  """simple docstring"""





  yield (time.time(), content)
  time.sleep(2		)
  yield (time.time(), content)






def 					lowerCAmelCase_			(	)					->		List[Any]:







  """simple docstring"""





  with Pool(2		) as pool:
    lowerCamelCase__:				Any        =list(iflatmap_unordered(__a     ,   _split_text     ,   kwargs_iterable=[{"text": "hello there"}] * 10		)		)
    assert out.count("hello"		) == 10
    assert out.count("there"		) == 10
    assert len(__a		) == 20

  # check multiprocess from pathos (uses dill for pickling)
  with multiprocess.Pool(2		) as pool:
    lowerCamelCase__:				Optional[int]        =list(iflatmap_unordered(__a     ,   _split_text     ,   kwargs_iterable=[{"text": "hello there"}] * 10		)		)
    assert out.count("hello"		) == 10
    assert out.count("there"		) == 10
    assert len(__a		) == 20

  # check that we get items as fast as possible
  with Pool(2		) as pool:
    lowerCamelCase__:				str        =[]
    for yield_time, content in iflatmap_unordered(
        __a     ,   _aseconds_generator_of_aitems_with_timing     ,   kwargs_iterable=[{"content": "a"}, {"content": "b"}]		):
      assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
      out.append(__a		)
    assert out.count("a"		) == 2
    assert out.count("b"		) == 2
    assert len(__a		) == 4








from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand





def 					lowerCAmelCase_			(	__a		)					->		int:







  """simple docstring"""





  return ConvertCommand(
      args.model_type     ,   args.tf_checkpoint     ,   args.pytorch_dump_output     ,   args.config     ,   args.finetuning_task_name		)


__A     =    "\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n"






class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''

 @staticmethod
 def 							SCREAMING_SNAKE_CASE_ (UpperCAmelCase_    :       ArgumentParser)	->str:



   '''simple docstring'''



   lowerCamelCase__:				Dict        =parser.add_parser(
       "convert"	,			help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."	,			)
   train_parser.add_argument("--model_type"	,			type=UpperCAmelCase_	,			required=UpperCAmelCase_	,			help="Model's type.")
   train_parser.add_argument(
       "--tf_checkpoint"	,			type=UpperCAmelCase_	,			required=UpperCAmelCase_	,			help="TensorFlow checkpoint path or folder.")
   train_parser.add_argument(
       "--pytorch_dump_output"	,			type=UpperCAmelCase_	,			required=UpperCAmelCase_	,			help="Path to the PyTorch saved model output.")
   train_parser.add_argument("--config"	,			type=UpperCAmelCase_	,			default=""	,			help="Configuration file path or folder.")
   train_parser.add_argument(
       "--finetuning_task_name"	,			type=UpperCAmelCase_	,			default=UpperCAmelCase_	,			help="Optional fine-tuning task name if the TF model was a finetuned model."	,			)
   train_parser.set_defaults(func=UpperCAmelCase_)

 def __init__(self    :       List[Any]	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			*UpperCAmelCase_    :       Optional[int]	,			)	->List[str]:



   '''simple docstring'''



   lowerCamelCase__:				Dict        =logging.get_logger("transformers-cli/converting")

   self._logger.info(F"""Loading model {model_type}""")
   lowerCamelCase__:				Any        =model_type
   lowerCamelCase__:				Optional[int]        =tf_checkpoint
   lowerCamelCase__:				Any        =pytorch_dump_output
   lowerCamelCase__:				Union[str, Any]        =config
   lowerCamelCase__:				str        =finetuning_task_name







 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any])	->Tuple:



   '''simple docstring'''



   if self._model_type == "albert":
     try:
       from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
           convert_tf_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "bert":
     try:
       from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
           convert_tf_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "funnel":
     try:
       from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
           convert_tf_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "t5":
     try:
       from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "gpt":
     from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
         convert_openai_checkpoint_to_pytorch,
     )

     convert_openai_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "transfo_xl":
     try:
       from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
           convert_transfo_xl_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     if "ckpt" in self._tf_checkpoint.lower():
       lowerCamelCase__:				Tuple        =self._tf_checkpoint
       lowerCamelCase__:				List[str]        =""
     else:
       lowerCamelCase__:				Any        =self._tf_checkpoint
       lowerCamelCase__:				Dict        =""
     convert_transfo_xl_checkpoint_to_pytorch(
         UpperCAmelCase_	,			self._config	,			self._pytorch_dump_output	,			UpperCAmelCase_)
   elif self._model_type == "gpt2":
     try:
       from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
           convert_gpta_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "xlnet":
     try:
       from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
           convert_xlnet_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_xlnet_checkpoint_to_pytorch(
         self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output	,			self._finetuning_task_name)
   elif self._model_type == "xlm":
     from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
         convert_xlm_checkpoint_to_pytorch,
     )

     convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint	,			self._pytorch_dump_output)
   elif self._model_type == "lxmert":
     from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
         convert_lxmert_checkpoint_to_pytorch,
     )

     convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint	,			self._pytorch_dump_output)
   elif self._model_type == "rembert":
     from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
         convert_rembert_tf_checkpoint_to_pytorch,
     )

     convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   else:
     raise ValueError(
         "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]")

