
import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch
def 					lowerCamelCase__		(				_lowerCamelCase  )						->int:  # picklable for multiprocessing
 return x.sum()
def 					lowerCamelCase__		(				_lowerCamelCase  )						->Optional[int]:  # picklable for multiprocessing
 return i + 1





@dataclass
class 						_a   :

       """simple docstring"""





       snake_case				=4_2
       snake_case				=4_2







class 						_a   (						a__       ):

       """simple docstring"""





       def     SCREAMING_SNAKE_CASE  (					self	):
        _UpperCAmelCase 				={}
        _UpperCAmelCase 				=[]
        _UpperCAmelCase 				=1
        _UpperCAmelCase 				=[1, 2]
        _UpperCAmelCase 				={"""a""": 1, """b""": 2}
        _UpperCAmelCase 				={"""a""": [1, 2], """b""": [3, 4]}
        _UpperCAmelCase 				={"""a""": {"""1""": 1}, """b""": 2}
        _UpperCAmelCase 				={"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
        _UpperCAmelCase 				={}
        _UpperCAmelCase 				=[]
        _UpperCAmelCase 				=2
        _UpperCAmelCase 				=[2, 3]
        _UpperCAmelCase 				={"""a""": 2, """b""": 3}
        _UpperCAmelCase 				={"""a""": [2, 3], """b""": [4, 5]}
        _UpperCAmelCase 				={"""a""": {"""1""": 2}, """b""": 3}
        _UpperCAmelCase 				={"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a	)       ,					__a	)

        _UpperCAmelCase 				=2
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(map_nested(__a       ,					__a       ,					num_proc=__a	)       ,					__a	)

        _UpperCAmelCase 				={"""a""": np.eye(2	), """b""": np.zeros(3	), """c""": np.ones(2	)}
        _UpperCAmelCase 				={"""a""": 2, """b""": 0, """c""": 2}
        _UpperCAmelCase 				={
            """a""": np.eye(2	).astype(__a	),
            """b""": np.zeros(3	).astype(__a	),
            """c""": np.ones(2	).astype(__a	),
        }
        self.assertEqual(map_nested(__a       ,					__a       ,					map_numpy=__a	)       ,					__a	)
        self.assertEqual(
            {k: v.tolist() for k, v in map_nested(__a       ,					__a       ,					map_numpy=__a	).items()}       ,					{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}       ,					)
        self.assertEqual(map_nested(__a       ,					__a       ,					map_numpy=__a       ,					num_proc=__a	)       ,					__a	)
        self.assertEqual(
            {k: v.tolist() for k, v in map_nested(__a       ,					__a       ,					map_numpy=__a       ,					num_proc=__a	).items()}       ,					{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}       ,					)
        with self.assertRaises(__a	):  # can't pickle a local lambda
         map_nested(lambda _snake_case	: x + 1       ,					__a       ,					num_proc=__a	)
       def     SCREAMING_SNAKE_CASE  (					self	):
        _UpperCAmelCase 				={"""a""": 1, """b""": 2}
        _UpperCAmelCase 				={"""a""": 3, """b""": 4}
        _UpperCAmelCase 				={"""a""": 5, """b""": 6}
        _UpperCAmelCase 				=sorted([("a", (1, 3, 5)), ("b", (2, 4, 6))]	)
        self.assertEqual(sorted(zip_dict(__a       ,					__a       ,					__a	)	)       ,					__a	)

       def     SCREAMING_SNAKE_CASE  (					self	):





        class 						_a   :

               """simple docstring"""





               snake_case				="""bar"""







        _UpperCAmelCase 				=Foo()
        self.assertEqual(foo.my_attr       ,					"bar"	)
        with temporary_assignment(__a       ,					"my_attr"       ,					"BAR"	):
         self.assertEqual(foo.my_attr       ,					"BAR"	)
        self.assertEqual(foo.my_attr       ,					"bar"	)







@pytest.mark.parametrize(
    "iterable_length, num_proc, expected_num_proc"			,		[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]			,		)
def 					lowerCamelCase__		(				_lowerCamelCase			,		_lowerCamelCase			,		_lowerCamelCase  )						->int:
 with patch("datasets.utils.py_utils._single_map_nested"  ) as mock_single_map_nested, patch(
     "datasets.parallel.parallel.Pool"  ) as mock_multiprocessing_pool:
  _UpperCAmelCase 				={F"{i}": i for i in range(A__  )}
  _UpperCAmelCase 				=map_nested(lambda _lowerCamelCase  : x + 10			,		A__			,		num_proc=A__			,		parallel_min_length=16  )
  if expected_num_proc == 1:
   assert mock_single_map_nested.called
   assert not mock_multiprocessing_pool.called
  else:
   assert not mock_single_map_nested.called
   assert mock_multiprocessing_pool.called
   assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc





class 						_a   (						a__       ):

       """simple docstring"""





       @require_tf
       def     SCREAMING_SNAKE_CASE  (					self	):
        import tensorflow as tf
        from tensorflow.keras import layers

        _UpperCAmelCase 				=layers.Dense(2	)

        def gen_random_output():
         _UpperCAmelCase 				=tf.random.uniform((1, 3)	)
         return model(__a	).numpy()

        with temp_seed(42       ,					set_tensorflow=__a	):
         _UpperCAmelCase 				=gen_random_output()
        with temp_seed(42       ,					set_tensorflow=__a	):
         _UpperCAmelCase 				=gen_random_output()
        _UpperCAmelCase 				=gen_random_output()

        np.testing.assert_equal(__a       ,					__a	)
        self.assertGreater(np.abs(outa - outa	).sum()       ,					0	)
       @require_torch
       def     SCREAMING_SNAKE_CASE  (					self	):
        import torch

        def gen_random_output():
         _UpperCAmelCase 				=torch.nn.Linear(3       ,					2	)
         _UpperCAmelCase 				=torch.rand(1       ,					3	)
         return model(__a	).detach().numpy()

        with temp_seed(42       ,					set_pytorch=__a	):
         _UpperCAmelCase 				=gen_random_output()
        with temp_seed(42       ,					set_pytorch=__a	):
         _UpperCAmelCase 				=gen_random_output()
        _UpperCAmelCase 				=gen_random_output()

        np.testing.assert_equal(__a       ,					__a	)
        self.assertGreater(np.abs(outa - outa	).sum()       ,					0	)

       def     SCREAMING_SNAKE_CASE  (					self	):
        def gen_random_output():
         return np.random.rand(1       ,					3	)

        with temp_seed(42	):
         _UpperCAmelCase 				=gen_random_output()
        with temp_seed(42	):
         _UpperCAmelCase 				=gen_random_output()
        _UpperCAmelCase 				=gen_random_output()

        np.testing.assert_equal(__a       ,					__a	)
        self.assertGreater(np.abs(outa - outa	).sum()       ,					0	)







@pytest.mark.parametrize("input_data"			,		[{}]  )
def 					lowerCamelCase__		(				_lowerCamelCase  )						->List[Any]:
 _UpperCAmelCase 				=NestedDataStructure(A__  ).data
 assert output_data == input_data
@pytest.mark.parametrize(
    "data, expected_output"			,		[
        ({}, []),
        ([], []),
        ("foo", ["foo"]),
        (["foo", "bar"], ["foo", "bar"]),
        ([["foo", "bar"]], ["foo", "bar"]),
        ([[["foo"], ["bar"]]], ["foo", "bar"]),
        ([[["foo"], "bar"]], ["foo", "bar"]),
        ({"a": 1, "b": 2}, [1, 2]),
        ({"a": [1, 2], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[1, 2]], "b": [[3, 4]]}, [1, 2, 3, 4]),
        ({"a": [[1, 2]], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [[3, 4]]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [3, [4]]}, [1, 2, 3, 4]),
        ({"a": {"1": 1}, "b": 2}, [1, 2]),
        ({"a": {"1": [1]}, "b": 2}, [1, 2]),
        ({"a": {"1": [1]}, "b": [2]}, [1, 2]),
    ]			,		)
def 					lowerCamelCase__		(				_lowerCamelCase			,		_lowerCamelCase  )						->Optional[Any]:
 _UpperCAmelCase 				=NestedDataStructure(A__  ).flatten()
 assert output == expected_output
def 					lowerCamelCase__		(				)						->Optional[Any]:
 _UpperCAmelCase 				=A(x=1			,		y="foobar"  )
 _UpperCAmelCase 				={"""x""": 1, """y""": """foobar"""}
 assert asdict(A__  ) == expected_output

 _UpperCAmelCase 				={"""a""": {"""b""": A(x=10			,		y="foo"  )}, """c""": [A(x=20			,		y="bar"  )]}
 _UpperCAmelCase 				={"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
 assert asdict(A__  ) == expected_output

 with pytest.raises(A__  ):
  asdict([1, A(x=10			,		y="foo"  )]  )
def 					lowerCamelCase__		(				_lowerCamelCase  )						->Any:
 return text.split()
def 					lowerCamelCase__		(				_lowerCamelCase  )						->List[Any]:
 yield (time.time(), content)
 time.sleep(2  )
 yield (time.time(), content)



def 					lowerCamelCase__		(				)						->List[str]:
 with Pool(2  ) as pool:
  _UpperCAmelCase 				=list(iflatmap_unordered(A__			,		_split_text			,		kwargs_iterable=[{"text": "hello there"}] * 10  )  )
  assert out.count("hello"  ) == 10
  assert out.count("there"  ) == 10
  assert len(A__  ) == 20

 # check multiprocess from pathos (uses dill for pickling)
 with multiprocess.Pool(2  ) as pool:
  _UpperCAmelCase 				=list(iflatmap_unordered(A__			,		_split_text			,		kwargs_iterable=[{"text": "hello there"}] * 10  )  )
  assert out.count("hello"  ) == 10
  assert out.count("there"  ) == 10
  assert len(A__  ) == 20

 # check that we get items as fast as possible
 with Pool(2  ) as pool:
  _UpperCAmelCase 				=[]
  for yield_time, content in iflatmap_unordered(
      A__			,		_aseconds_generator_of_aitems_with_timing			,		kwargs_iterable=[{"content": "a"}, {"content": "b"}]  ):
   assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
   out.append(A__  )
  assert out.count("a"  ) == 2
  assert out.count("b"  ) == 2
  assert len(A__  ) == 4









from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__				:	List[Any]					       =					logging.get_logger(__name__)

snake_case__				:	str					       =					{
    'facebook/dpr-ctx_encoder-single-nq-base': (
        'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json'
    ),
    'facebook/dpr-question_encoder-single-nq-base': (
        'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json'
    ),
    'facebook/dpr-reader-single-nq-base': (
        'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json'
    ),
    'facebook/dpr-ctx_encoder-multiset-base': (
        'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json'
    ),
    'facebook/dpr-question_encoder-multiset-base': (
        'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json'
    ),
    'facebook/dpr-reader-multiset-base': (
        'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json'
    ),
}





class   SCREAMING_SNAKE_CASE_			(a__						):




       '''simple docstring'''



       _a   							=							"dpr"



       def __init__(						self				:				Union[str, Any]		,				__a				:				Optional[Any]=30_522		,				__a				:				List[Any]=768		,				__a				:				List[Any]=12		,				__a				:				Dict=12		,				__a				:				Union[str, Any]=3_072		,				__a				:				Any="gelu"		,				__a				:				Any=0.1		,				__a				:				Any=0.1		,				__a				:				Tuple=512		,				__a				:				int=2		,				__a				:				Optional[Any]=0.02		,				__a				:				List[Any]=1e-12		,				__a				:				int=0		,				__a				:				int="absolute"		,				__a				:				int = 0		,				**__a				:				Optional[int]		,				)					->Tuple:
          super().__init__(pad_token_id=__a		,				**__a							)

          lowerCamelCase_						:   str		= vocab_size
          lowerCamelCase_						:   List[Any]		= hidden_size
          lowerCamelCase_						:   Optional[int]		= num_hidden_layers
          lowerCamelCase_						:   Optional[Any]		= num_attention_heads
          lowerCamelCase_						:   List[Any]		= hidden_act
          lowerCamelCase_						:   Optional[Any]		= intermediate_size
          lowerCamelCase_						:   List[Any]		= hidden_dropout_prob
          lowerCamelCase_						:   Any		= attention_probs_dropout_prob
          lowerCamelCase_						:   Any		= max_position_embeddings
          lowerCamelCase_						:   Tuple		= type_vocab_size
          lowerCamelCase_						:   Dict		= initializer_range
          lowerCamelCase_						:   List[Any]		= layer_norm_eps
          lowerCamelCase_						:   List[str]		= projection_dim
          lowerCamelCase_						:   Optional[int]		= position_embedding_type





