

import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch




def 					_lowercase     (				a_				:					List[Any]				)				->	Optional[Any]:  # picklable for multiprocessing




					'''simple docstring'''







					return x.sum()




def 					_lowercase     (				a_				:					Optional[Any]				)				->	Optional[Any]:  # picklable for multiprocessing




					'''simple docstring'''







					return i + 1

@dataclass
class 				__UpperCamelCase				:
	_lowercase  :   int
	_lowercase  :   str




class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):







	def 	_SCREAMING_SNAKE_CASE							(   self: Any      ):

						'''simple docstring'''





						__magic_name__							=    {}
						__magic_name__							=    []
						__magic_name__							=    1
						__magic_name__							=    [1, 2]
						__magic_name__							=    {'a': 1, 'b': 2}
						__magic_name__							=    {'a': [1, 2], 'b': [3, 4]}
						__magic_name__							=    {'a': {'1': 1}, 'b': 2}
						__magic_name__							=    {'a': 1, 'b': 2, 'c': 3, 'd': 4}
						__magic_name__							=    {}
						__magic_name__							=    []
						__magic_name__							=    2
						__magic_name__							=    [2, 3]
						__magic_name__							=    {'a': 2, 'b': 3}
						__magic_name__							=    {'a': [2, 3], 'b': [4, 5]}
						__magic_name__							=    {'a': {'1': 2}, 'b': 3}
						__magic_name__							=    {'a': 2, 'b': 3, 'c': 4, 'd': 5}
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase      )  , __UpperCamelCase      )

						__magic_name__							=    2
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )

						__magic_name__							=    {'a': np.eye(2      ), 'b': np.zeros(3      ), 'c': np.ones(2      )}
						__magic_name__							=    {'a': 2, 'b': 0, 'c': 2}
						__magic_name__							=    {
						    'a': np.eye(2      ).astype(__UpperCamelCase      ),
						    'b': np.zeros(3      ).astype(__UpperCamelCase      ),
						    'c': np.ones(2      ).astype(__UpperCamelCase      ),
						}
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , map_numpy=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(
						    {k: v.tolist() for k, v in map_nested(__UpperCamelCase  , __UpperCamelCase  , map_numpy=__UpperCamelCase      ).items()}  , {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}  , )
						self.assertEqual(map_nested(__UpperCamelCase  , __UpperCamelCase  , map_numpy=__UpperCamelCase  , num_proc=__UpperCamelCase      )  , __UpperCamelCase      )
						self.assertEqual(
						    {k: v.tolist() for k, v in map_nested(__UpperCamelCase  , __UpperCamelCase  , map_numpy=__UpperCamelCase  , num_proc=__UpperCamelCase      ).items()}  , {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}  , )
						with self.assertRaises(__UpperCamelCase      ):  # can't pickle a local lambda
											map_nested(lambda __UpperCamelCase      : x + 1  , __UpperCamelCase  , num_proc=__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple      ):

						'''simple docstring'''





						__magic_name__							=    {'a': 1, 'b': 2}
						__magic_name__							=    {'a': 3, 'b': 4}
						__magic_name__							=    {'a': 5, 'b': 6}
						__magic_name__							=    sorted([('a', (1, 3, 5)), ('b', (2, 4, 6))]      )
						self.assertEqual(sorted(zip_dict(__UpperCamelCase  , __UpperCamelCase  , __UpperCamelCase      )      )  , __UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						class 				__UpperCamelCase				:
							_lowercase  :   Optional[int]       				=  "bar"




						__magic_name__							=    Foo()
						self.assertEqual(foo.my_attr  , 'bar'      )
						with temporary_assignment(__UpperCamelCase  , 'my_attr'  , 'BAR'      ):
											self.assertEqual(foo.my_attr  , 'BAR'      )
						self.assertEqual(foo.my_attr  , 'bar'      )




@pytest.mark.parametrize(
    'iterable_length, num_proc, expected_num_proc'					,[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (1_6, 1_6, 1_6),
        (1_6, 1_7, 1_6),
        (1_7, 1_6, 1_6),
    ]					,)
def 					_lowercase     (				a_				:					Optional[Any]					,a_				:					int					,a_				:					Optional[Any]				)				->	Any:




					'''simple docstring'''







					with patch('datasets.utils.py_utils._single_map_nested'				) as mock_single_map_nested, patch(
					    'datasets.parallel.parallel.Pool'				) as mock_multiprocessing_pool:
										__magic_name__							=    {F'{i}': i for i in range(a_				)}
										__magic_name__							=    map_nested(lambda a_				: x + 1_0					,a_					,num_proc=a_					,parallel_min_length=1_6				)
										if expected_num_proc == 1:
															assert mock_single_map_nested.called
															assert not mock_multiprocessing_pool.called
										else:
															assert not mock_single_map_nested.called
															assert mock_multiprocessing_pool.called
															assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):







	@require_tf
	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple      ):

						'''simple docstring'''





						import tensorflow as tf
						from tensorflow.keras import layers

						__magic_name__							=    layers.Dense(2      )

						def gen_random_output():
											__magic_name__							=    tf.random.uniform((1, 3)      )
											return model(__UpperCamelCase      ).numpy()

						with temp_seed(42  , set_tensorflow=__UpperCamelCase      ):
											__magic_name__							=    gen_random_output()
						with temp_seed(42  , set_tensorflow=__UpperCamelCase      ):
											__magic_name__							=    gen_random_output()
						__magic_name__							=    gen_random_output()

						np.testing.assert_equal(__UpperCamelCase  , __UpperCamelCase      )
						self.assertGreater(np.abs(outa - outa      ).sum()  , 0      )







	@require_torch
	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						import torch

						def gen_random_output():
											__magic_name__							=    torch.nn.Linear(3  , 2      )
											__magic_name__							=    torch.rand(1  , 3      )
											return model(__UpperCamelCase      ).detach().numpy()

						with temp_seed(42  , set_pytorch=__UpperCamelCase      ):
											__magic_name__							=    gen_random_output()
						with temp_seed(42  , set_pytorch=__UpperCamelCase      ):
											__magic_name__							=    gen_random_output()
						__magic_name__							=    gen_random_output()

						np.testing.assert_equal(__UpperCamelCase  , __UpperCamelCase      )
						self.assertGreater(np.abs(outa - outa      ).sum()  , 0      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]      ):

						'''simple docstring'''





						def gen_random_output():
											return np.random.rand(1  , 3      )

						with temp_seed(42      ):
											__magic_name__							=    gen_random_output()
						with temp_seed(42      ):
											__magic_name__							=    gen_random_output()
						__magic_name__							=    gen_random_output()

						np.testing.assert_equal(__UpperCamelCase  , __UpperCamelCase      )
						self.assertGreater(np.abs(outa - outa      ).sum()  , 0      )




@pytest.mark.parametrize('input_data'					,[{}]				)
def 					_lowercase     (				a_				:					Optional[int]				)				->	Dict:




					'''simple docstring'''







					__magic_name__							=    NestedDataStructure(a_				).data
					assert output_data == input_data




@pytest.mark.parametrize(
    'data, expected_output'					,[
        ({}, []),
        ([], []),
        ('foo', ['foo']),
        (['foo', 'bar'], ['foo', 'bar']),
        ([['foo', 'bar']], ['foo', 'bar']),
        ([[['foo'], ['bar']]], ['foo', 'bar']),
        ([[['foo'], 'bar']], ['foo', 'bar']),
        ({'a': 1, 'b': 2}, [1, 2]),
        ({'a': [1, 2], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, [4]]}, [1, 2, 3, 4]),
        ({'a': {'1': 1}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': [2]}, [1, 2]),
    ]					,)
def 					_lowercase     (				a_				:					int					,a_				:					Tuple				)				->	List[str]:




					'''simple docstring'''







					__magic_name__							=    NestedDataStructure(a_				).flatten()
					assert output == expected_output




def 					_lowercase     (				)				->	Tuple:




					'''simple docstring'''







					__magic_name__							=    A(x=1					,y='foobar'				)
					__magic_name__							=    {'x': 1, 'y': 'foobar'}
					assert asdict(a_				) == expected_output

					__magic_name__							=    {'a': {'b': A(x=1_0					,y='foo'				)}, 'c': [A(x=2_0					,y='bar'				)]}
					__magic_name__							=    {'a': {'b': {'x': 1_0, 'y': 'foo'}}, 'c': [{'x': 2_0, 'y': 'bar'}]}
					assert asdict(a_				) == expected_output

					with pytest.raises(a_				):
										asdict([1, A(x=1_0					,y='foo'				)]				)




def 					_lowercase     (				a_				:					str				)				->	Optional[int]:




					'''simple docstring'''







					return text.split()




def 					_lowercase     (				a_				:					List[str]				)				->	Dict:




					'''simple docstring'''







					yield (time.time(), content)
					time.sleep(2				)
					yield (time.time(), content)




def 					_lowercase     (				)				->	List[str]:




					'''simple docstring'''







					with Pool(2				) as pool:
										__magic_name__							=    list(iflatmap_unordered(a_					,_split_text					,kwargs_iterable=[{'text': 'hello there'}] * 1_0				)				)
										assert out.count('hello'				) == 1_0
										assert out.count('there'				) == 1_0
										assert len(a_				) == 2_0

					# check multiprocess from pathos (uses dill for pickling)
					with multiprocess.Pool(2				) as pool:
										__magic_name__							=    list(iflatmap_unordered(a_					,_split_text					,kwargs_iterable=[{'text': 'hello there'}] * 1_0				)				)
										assert out.count('hello'				) == 1_0
										assert out.count('there'				) == 1_0
										assert len(a_				) == 2_0

					# check that we get items as fast as possible
					with Pool(2				) as pool:
										__magic_name__							=    []
										for yield_time, content in iflatmap_unordered(
										    a_					,_aseconds_generator_of_aitems_with_timing					,kwargs_iterable=[{'content': 'a'}, {'content': 'b'}]				):
															assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
															out.append(a_				)
										assert out.count('a'				) == 2
										assert out.count('b'				) == 2
										assert len(a_				) == 4







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


if TYPE_CHECKING:
						from ...processing_utils import ProcessorMixin
						from ...utils import TensorType


A__						=							logging.get_logger(__name__)

A__						=							{
    "microsoft/layoutlmv3-base": "https://huggingface.co/microsoft/layoutlmv3-base/resolve/main/config.json",
}

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):
	_lowercase  :   Optional[Any]       				=  "layoutlmv3"







	def __init__(   self: Tuple  , __UpperCamelCase: List[Any]=5_02_65  , __UpperCamelCase: Union[str, Any]=7_68  , __UpperCamelCase: Optional[int]=12  , __UpperCamelCase: List[str]=12  , __UpperCamelCase: List[Any]=30_72  , __UpperCamelCase: Union[str, Any]="gelu"  , __UpperCamelCase: Union[str, Any]=0.1  , __UpperCamelCase: Union[str, Any]=0.1  , __UpperCamelCase: List[Any]=5_12  , __UpperCamelCase: int=2  , __UpperCamelCase: Union[str, Any]=0.02  , __UpperCamelCase: Dict=1E-5  , __UpperCamelCase: int=1  , __UpperCamelCase: Dict=0  , __UpperCamelCase: Tuple=2  , __UpperCamelCase: Any=10_24  , __UpperCamelCase: Union[str, Any]=1_28  , __UpperCamelCase: Dict=1_28  , __UpperCamelCase: Any=True  , __UpperCamelCase: Any=32  , __UpperCamelCase: int=1_28  , __UpperCamelCase: List[str]=64  , __UpperCamelCase: Union[str, Any]=2_56  , __UpperCamelCase: int=True  , __UpperCamelCase: Optional[int]=True  , __UpperCamelCase: Union[str, Any]=True  , __UpperCamelCase: Union[str, Any]=2_24  , __UpperCamelCase: Tuple=3  , __UpperCamelCase: Any=16  , __UpperCamelCase: List[Any]=None  , **__UpperCamelCase: str  , ):

						'''simple docstring'''





						super().__init__(
						    vocab_size=__UpperCamelCase  , hidden_size=__UpperCamelCase  , num_hidden_layers=__UpperCamelCase  , num_attention_heads=__UpperCamelCase  , intermediate_size=__UpperCamelCase  , hidden_act=__UpperCamelCase  , hidden_dropout_prob=__UpperCamelCase  , attention_probs_dropout_prob=__UpperCamelCase  , max_position_embeddings=__UpperCamelCase  , type_vocab_size=__UpperCamelCase  , initializer_range=__UpperCamelCase  , layer_norm_eps=__UpperCamelCase  , pad_token_id=__UpperCamelCase  , bos_token_id=__UpperCamelCase  , eos_token_id=__UpperCamelCase  , **__UpperCamelCase  , )
						__magic_name__							=    max_ad_position_embeddings
						__magic_name__							=    coordinate_size
						__magic_name__							=    shape_size
						__magic_name__							=    has_relative_attention_bias
						__magic_name__							=    rel_pos_bins
						__magic_name__							=    max_rel_pos
						__magic_name__							=    has_spatial_attention_bias
						__magic_name__							=    rel_ad_pos_bins
						__magic_name__							=    max_rel_ad_pos
						__magic_name__							=    text_embed
						__magic_name__							=    visual_embed
						__magic_name__							=    input_size
						__magic_name__							=    num_channels
						__magic_name__							=    patch_size
						__magic_name__							=    classifier_dropout




class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):
	_lowercase  :   Any       				=  version.parse("1.12"			)







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: Any      ):

						'''simple docstring'''





						if self.task in ["question-answering", "sequence-classification"]:
											return OrderedDict(
											    [
											        ('input_ids', {0: 'batch', 1: 'sequence'}),
											        ('attention_mask', {0: 'batch', 1: 'sequence'}),
											        ('bbox', {0: 'batch', 1: 'sequence'}),
											        ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
											    ]      )
						else:
											return OrderedDict(
											    [
											        ('input_ids', {0: 'batch', 1: 'sequence'}),
											        ('bbox', {0: 'batch', 1: 'sequence'}),
											        ('attention_mask', {0: 'batch', 1: 'sequence'}),
											        ('pixel_values', {0: 'batch', 1: 'num_channels'}),
											    ]      )







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: int      ):

						'''simple docstring'''





						return 1E-5







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]      ):

						'''simple docstring'''





						return 12







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: "ProcessorMixin"  , __UpperCamelCase: int = -1  , __UpperCamelCase: int = -1  , __UpperCamelCase: bool = False  , __UpperCamelCase: Optional["TensorType"] = None  , __UpperCamelCase: int = 3  , __UpperCamelCase: int = 40  , __UpperCamelCase: int = 40  , ):

						'''simple docstring'''





						setattr(processor.image_processor  , 'apply_ocr'  , __UpperCamelCase      )

						# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
						__magic_name__							=    compute_effective_axis_dimension(
						    __UpperCamelCase  , fixed_dimension=OnnxConfig.default_fixed_batch  , num_token_to_add=0      )
						# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
						__magic_name__							=    processor.tokenizer.num_special_tokens_to_add(__UpperCamelCase      )
						__magic_name__							=    compute_effective_axis_dimension(
						    __UpperCamelCase  , fixed_dimension=OnnxConfig.default_fixed_sequence  , num_token_to_add=__UpperCamelCase      )
						# Generate dummy inputs according to compute batch and sequence
						__magic_name__							=    [[' '.join([processor.tokenizer.unk_token]      ) * seq_length]] * batch_size

						# Generate dummy bounding boxes
						__magic_name__							=    [[[48, 84, 73, 1_28]]] * batch_size

						# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
						# batch_size = compute_effective_axis_dimension(batch_size, fixed_dimension=OnnxConfig.default_fixed_batch)
						__magic_name__							=    self._generate_dummy_images(__UpperCamelCase  , __UpperCamelCase  , __UpperCamelCase  , __UpperCamelCase      )

						__magic_name__							=    dict(
						    processor(
						        __UpperCamelCase  , text=__UpperCamelCase  , boxes=__UpperCamelCase  , return_tensors=__UpperCamelCase  , )      )

						return inputs




