


"""simple docstring"""




import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch





def       a_ (	lowercase__ :Union[str, Any]     ):  # picklable for multiprocessing
			return x.sum()





def       a_ (	lowercase__ :Dict     ):  # picklable for multiprocessing
			return i + 1




@dataclass
class 			__snake_case   :
							__a					 =       42
							__a					 =       42




class 			__snake_case   (lowerCamelCase  ):




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      {}
										__lowerCamelCase          =      []
										__lowerCamelCase          =      1
										__lowerCamelCase          =      [1, 2]
										__lowerCamelCase          =      {"""a""": 1, """b""": 2}
										__lowerCamelCase          =      {"""a""": [1, 2], """b""": [3, 4]}
										__lowerCamelCase          =      {"""a""": {"""1""": 1}, """b""": 2}
										__lowerCamelCase          =      {"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
										__lowerCamelCase          =      {}
										__lowerCamelCase          =      []
										__lowerCamelCase          =      2
										__lowerCamelCase          =      [2, 3]
										__lowerCamelCase          =      {"""a""": 2, """b""": 3}
										__lowerCamelCase          =      {"""a""": [2, 3], """b""": [4, 5]}
										__lowerCamelCase          =      {"""a""": {"""1""": 2}, """b""": 3}
										__lowerCamelCase          =      {"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_   )     ,							A_   )

										__lowerCamelCase          =      2
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(map_nested(A_     ,							A_     ,							num_proc=A_   )     ,							A_   )

										__lowerCamelCase          =      {"""a""": np.eye(2   ), """b""": np.zeros(3   ), """c""": np.ones(2   )}
										__lowerCamelCase          =      {"""a""": 2, """b""": 0, """c""": 2}
										__lowerCamelCase          =      {
										    """a""": np.eye(2   ).astype(A_   ),
										    """b""": np.zeros(3   ).astype(A_   ),
										    """c""": np.ones(2   ).astype(A_   ),
										}
										self.assertEqual(map_nested(A_     ,							A_     ,							map_numpy=A_   )     ,							A_   )
										self.assertEqual(
										    {k: v.tolist() for k, v in map_nested(A_     ,							A_     ,							map_numpy=A_   ).items()}     ,							{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}     ,							)
										self.assertEqual(map_nested(A_     ,							A_     ,							map_numpy=A_     ,							num_proc=A_   )     ,							A_   )
										self.assertEqual(
										    {k: v.tolist() for k, v in map_nested(A_     ,							A_     ,							map_numpy=A_     ,							num_proc=A_   ).items()}     ,							{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}     ,							)
										with self.assertRaises(A_   ):  # can't pickle a local lambda
													map_nested(lambda A_   : x + 1     ,							A_     ,							num_proc=A_   )




							def   __a   (     self:    Dict   ):
										__lowerCamelCase          =      {"""a""": 1, """b""": 2}
										__lowerCamelCase          =      {"""a""": 3, """b""": 4}
										__lowerCamelCase          =      {"""a""": 5, """b""": 6}
										__lowerCamelCase          =      sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))]   )
										self.assertEqual(sorted(zip_dict(A_     ,							A_     ,							A_   )   )     ,							A_   )




							def   __a   (     self:    Dict   ):




										class 			__snake_case   :
																	__a					 =       '''bar'''

										__lowerCamelCase          =      Foo()
										self.assertEqual(foo.my_attr     ,							"""bar"""   )
										with temporary_assignment(A_     ,							"""my_attr"""     ,							"""BAR"""   ):
													self.assertEqual(foo.my_attr     ,							"""BAR"""   )
										self.assertEqual(foo.my_attr     ,							"""bar"""   )





@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc""",			[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ],			)
def       a_ (	lowercase__ :int,			lowercase__ :Optional[int],			lowercase__ :Optional[Any]     ):
			with patch("""datasets.utils.py_utils._single_map_nested"""     ) as mock_single_map_nested, patch(
			    """datasets.parallel.parallel.Pool"""     ) as mock_multiprocessing_pool:
						__lowerCamelCase          =      {f'{i}': i for i in range(lowercase__     )}
						__lowerCamelCase          =      map_nested(lambda lowercase__     : x + 10,			lowercase__,			num_proc=lowercase__,			parallel_min_length=16     )
						if expected_num_proc == 1:
									assert mock_single_map_nested.called
									assert not mock_multiprocessing_pool.called
						else:
									assert not mock_single_map_nested.called
									assert mock_multiprocessing_pool.called
									assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc




class 			__snake_case   (lowerCamelCase  ):




							@require_tf
							def   __a   (     self:    str   ):
										import tensorflow as tf
										from tensorflow.keras import layers

										__lowerCamelCase          =      layers.Dense(2   )

										def gen_random_output():
													__lowerCamelCase          =      tf.random.uniform((1, 3)   )
													return model(A_   ).numpy()

										with temp_seed(42     ,							set_tensorflow=A_   ):
													__lowerCamelCase          =      gen_random_output()
										with temp_seed(42     ,							set_tensorflow=A_   ):
													__lowerCamelCase          =      gen_random_output()
										__lowerCamelCase          =      gen_random_output()

										np.testing.assert_equal(A_     ,							A_   )
										self.assertGreater(np.abs(outa - outa   ).sum()     ,							0   )




							@require_torch
							def   __a   (     self:    Tuple   ):
										import torch

										def gen_random_output():
													__lowerCamelCase          =      torch.nn.Linear(3     ,							2   )
													__lowerCamelCase          =      torch.rand(1     ,							3   )
													return model(A_   ).detach().numpy()

										with temp_seed(42     ,							set_pytorch=A_   ):
													__lowerCamelCase          =      gen_random_output()
										with temp_seed(42     ,							set_pytorch=A_   ):
													__lowerCamelCase          =      gen_random_output()
										__lowerCamelCase          =      gen_random_output()

										np.testing.assert_equal(A_     ,							A_   )
										self.assertGreater(np.abs(outa - outa   ).sum()     ,							0   )




							def   __a   (     self:    Tuple   ):
										def gen_random_output():
													return np.random.rand(1     ,							3   )

										with temp_seed(42   ):
													__lowerCamelCase          =      gen_random_output()
										with temp_seed(42   ):
													__lowerCamelCase          =      gen_random_output()
										__lowerCamelCase          =      gen_random_output()

										np.testing.assert_equal(A_     ,							A_   )
										self.assertGreater(np.abs(outa - outa   ).sum()     ,							0   )





@pytest.mark.parametrize("""input_data""",			[{}]     )
def       a_ (	lowercase__ :Dict     ):
			__lowerCamelCase          =      NestedDataStructure(lowercase__     ).data
			assert output_data == input_data





@pytest.mark.parametrize(
    """data, expected_output""",			[
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ],			)
def       a_ (	lowercase__ :Any,			lowercase__ :List[str]     ):
			__lowerCamelCase          =      NestedDataStructure(lowercase__     ).flatten()
			assert output == expected_output





def       a_ (	):
			__lowerCamelCase          =      A(x=1,			y="""foobar"""     )
			__lowerCamelCase          =      {"""x""": 1, """y""": """foobar"""}
			assert asdict(lowercase__     ) == expected_output

			__lowerCamelCase          =      {"""a""": {"""b""": A(x=10,			y="""foo"""     )}, """c""": [A(x=20,			y="""bar"""     )]}
			__lowerCamelCase          =      {"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
			assert asdict(lowercase__     ) == expected_output

			with pytest.raises(lowercase__     ):
						asdict([1, A(x=10,			y="""foo"""     )]     )





def       a_ (	lowercase__ :str     ):
			return text.split()





def       a_ (	lowercase__ :Optional[Any]     ):
			yield (time.time(), content)
			time.sleep(2     )
			yield (time.time(), content)





def       a_ (	):
			with Pool(2     ) as pool:
						__lowerCamelCase          =      list(iflatmap_unordered(lowercase__,			_split_text,			kwargs_iterable=[{"""text""": """hello there"""}] * 10     )     )
						assert out.count("""hello"""     ) == 10
						assert out.count("""there"""     ) == 10
						assert len(lowercase__     ) == 20

			# check multiprocess from pathos (uses dill for pickling)
			with multiprocess.Pool(2     ) as pool:
						__lowerCamelCase          =      list(iflatmap_unordered(lowercase__,			_split_text,			kwargs_iterable=[{"""text""": """hello there"""}] * 10     )     )
						assert out.count("""hello"""     ) == 10
						assert out.count("""there"""     ) == 10
						assert len(lowercase__     ) == 20

			# check that we get items as fast as possible
			with Pool(2     ) as pool:
						__lowerCamelCase          =      []
						for yield_time, content in iflatmap_unordered(
						    lowercase__,			_aseconds_generator_of_aitems_with_timing,			kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]     ):
									assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
									out.append(lowercase__     )
						assert out.count("""a"""     ) == 2
						assert out.count("""b"""     ) == 2
						assert len(lowercase__     ) == 4








"""simple docstring"""




import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType




class 			__snake_case   (lowerCamelCase  ):
							__a					 =       ['''image_processor''', '''tokenizer''']
							__a					 =       '''LayoutLMv2ImageProcessor'''
							__a					 =       ('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')




							def __init__(     self:    Any     ,							A_:    Optional[Any]=None     ,							A_:    Dict=None     ,							**A_:    Any   ):
										if "feature_extractor" in kwargs:
													warnings.warn(
													    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
													    """ instead."""     ,							A_     ,							)
													__lowerCamelCase          =      kwargs.pop("""feature_extractor"""   )

										__lowerCamelCase          =      image_processor if image_processor is not None else feature_extractor
										if image_processor is None:
													raise ValueError("""You need to specify an `image_processor`."""   )
										if tokenizer is None:
													raise ValueError("""You need to specify a `tokenizer`."""   )

										super().__init__(A_     ,							A_   )




							def __call__(     self:    List[str]     ,							A_:    Any     ,							A_:    Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None     ,							A_:    Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None     ,							A_:    Union[List[List[int]], List[List[List[int]]]] = None     ,							A_:    Optional[Union[List[int], List[List[int]]]] = None     ,							A_:    bool = True     ,							A_:    Union[bool, str, PaddingStrategy] = False     ,							A_:    Union[bool, str, TruncationStrategy] = None     ,							A_:    Optional[int] = None     ,							A_:    int = 0     ,							A_:    Optional[int] = None     ,							A_:    Optional[bool] = None     ,							A_:    Optional[bool] = None     ,							A_:    bool = False     ,							A_:    bool = False     ,							A_:    bool = False     ,							A_:    bool = False     ,							A_:    bool = True     ,							A_:    Optional[Union[str, TensorType]] = None     ,							**A_:    Any     ,							):
										# verify input
										if self.image_processor.apply_ocr and (boxes is not None):
													raise ValueError(
													    """You cannot provide bounding boxes """
													    """if you initialized the image processor with apply_ocr set to True."""   )

										if self.image_processor.apply_ocr and (word_labels is not None):
													raise ValueError(
													    """You cannot provide word labels if you initialized the image processor with apply_ocr set to True."""   )

										if return_overflowing_tokens is True and return_offsets_mapping is False:
													raise ValueError("""You cannot return overflowing tokens without returning the offsets mapping."""   )

										# first, apply the image processor
										__lowerCamelCase          =      self.image_processor(images=A_     ,							return_tensors=A_   )

										# second, apply the tokenizer
										if text is not None and self.image_processor.apply_ocr and text_pair is None:
													if isinstance(A_     ,							A_   ):
																__lowerCamelCase          =      [text]  # add batch dimension (as the image processor always adds a batch dimension)
													__lowerCamelCase          =      features["""words"""]

										__lowerCamelCase          =      self.tokenizer(
										    text=text if text is not None else features["""words"""]     ,							text_pair=text_pair if text_pair is not None else None     ,							boxes=boxes if boxes is not None else features["""boxes"""]     ,							word_labels=A_     ,							add_special_tokens=A_     ,							padding=A_     ,							truncation=A_     ,							max_length=A_     ,							stride=A_     ,							pad_to_multiple_of=A_     ,							return_token_type_ids=A_     ,							return_attention_mask=A_     ,							return_overflowing_tokens=A_     ,							return_special_tokens_mask=A_     ,							return_offsets_mapping=A_     ,							return_length=A_     ,							verbose=A_     ,							return_tensors=A_     ,							**A_     ,							)

										# add pixel values
										__lowerCamelCase          =      features.pop("""pixel_values"""   )
										if return_overflowing_tokens is True:
													__lowerCamelCase          =      self.get_overflowing_images(A_     ,							encoded_inputs["""overflow_to_sample_mapping"""]   )
										__lowerCamelCase          =      images

										return encoded_inputs




							def   __a   (     self:    Optional[Any]     ,							A_:    Any     ,							A_:    Tuple   ):
										# in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
										__lowerCamelCase          =      []
										for sample_idx in overflow_to_sample_mapping:
													images_with_overflow.append(images[sample_idx]   )

										if len(A_   ) != len(A_   ):
													raise ValueError(
													    """Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"""
													    f' {len(A_   )} and {len(A_   )}'   )

										return images_with_overflow




							def   __a   (     self:    Optional[Any]     ,							*A_:    str     ,							**A_:    Dict   ):
										return self.tokenizer.batch_decode(*A_     ,							**A_   )




							def   __a   (     self:    Union[str, Any]     ,							*A_:    Optional[Any]     ,							**A_:    Tuple   ):
										return self.tokenizer.decode(*A_     ,							**A_   )




							@property
							def   __a   (     self:    Union[str, Any]   ):
										return ["input_ids", "bbox", "attention_mask", "image"]




							@property
							def   __a   (     self:    Optional[Any]   ):
										warnings.warn(
										    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""     ,							A_     ,							)
										return self.image_processor_class




							@property
							def   __a   (     self:    List[Any]   ):
										warnings.warn(
										    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""     ,							A_     ,							)
										return self.image_processor




