
'''simple docstring'''
import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch







def 		A_(     A						:		str):  # picklable for multiprocessing
							return x.sum()







def 		A_(     A						:		Union[str, Any]):  # picklable for multiprocessing
							return i + 1


@dataclass
class 			SCREAMING_SNAKE_CASE__      :
				lowerCAmelCase_     			=							42
				lowerCAmelCase_     			=							42


class 			SCREAMING_SNAKE_CASE__      (     snake_case_):







				def 						UpperCAmelCase_    (       self     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     {}
											UpperCamelCase   			=     []
											UpperCamelCase   			=     1
											UpperCamelCase   			=     [1, 2]
											UpperCamelCase   			=     {'a': 1, 'b': 2}
											UpperCamelCase   			=     {'a': [1, 2], 'b': [3, 4]}
											UpperCamelCase   			=     {'a': {'1': 1}, 'b': 2}
											UpperCamelCase   			=     {'a': 1, 'b': 2, 'c': 3, 'd': 4}
											UpperCamelCase   			=     {}
											UpperCamelCase   			=     []
											UpperCamelCase   			=     2
											UpperCamelCase   			=     [2, 3]
											UpperCamelCase   			=     {'a': 2, 'b': 3}
											UpperCamelCase   			=     {'a': [2, 3], 'b': [4, 5]}
											UpperCamelCase   			=     {'a': {'1': 2}, 'b': 3}
											UpperCamelCase   			=     {'a': 2, 'b': 3, 'c': 4, 'd': 5}
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_     )						,				A_     )

											UpperCamelCase   			=     2
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(map_nested(A_						,				A_						,				num_proc=A_     )						,				A_     )

											UpperCamelCase   			=     {'a': np.eye(2     ), 'b': np.zeros(3     ), 'c': np.ones(2     )}
											UpperCamelCase   			=     {'a': 2, 'b': 0, 'c': 2}
											UpperCamelCase   			=     {
											    'a': np.eye(2     ).astype(A_     ),
											    'b': np.zeros(3     ).astype(A_     ),
											    'c': np.ones(2     ).astype(A_     ),
											}
											self.assertEqual(map_nested(A_						,				A_						,				map_numpy=A_     )						,				A_     )
											self.assertEqual(
											    {k: v.tolist() for k, v in map_nested(A_						,				A_						,				map_numpy=A_     ).items()}						,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}						,				)
											self.assertEqual(map_nested(A_						,				A_						,				map_numpy=A_						,				num_proc=A_     )						,				A_     )
											self.assertEqual(
											    {k: v.tolist() for k, v in map_nested(A_						,				A_						,				map_numpy=A_						,				num_proc=A_     ).items()}						,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}						,				)
											with self.assertRaises(A_     ):  # can't pickle a local lambda
																		map_nested(lambda A_     : x + 1						,				A_						,				num_proc=A_     )







				def 						UpperCAmelCase_    (       self     )->		Tuple:







											'''simple docstring'''






											UpperCamelCase   			=     {'a': 1, 'b': 2}
											UpperCamelCase   			=     {'a': 3, 'b': 4}
											UpperCamelCase   			=     {'a': 5, 'b': 6}
											UpperCamelCase   			=     sorted([('a', (1, 3, 5)), ('b', (2, 4, 6))]     )
											self.assertEqual(sorted(zip_dict(A_						,				A_						,				A_     )     )						,				A_     )







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											class 			SCREAMING_SNAKE_CASE__      :
															lowerCAmelCase_     			=							"""bar"""


											UpperCamelCase   			=     Foo()
											self.assertEqual(foo.my_attr						,				'bar'     )
											with temporary_assignment(A_						,				'my_attr'						,				'BAR'     ):
																		self.assertEqual(foo.my_attr						,				'BAR'     )
											self.assertEqual(foo.my_attr						,				'bar'     )







@pytest.mark.parametrize(
    'iterable_length, num_proc, expected_num_proc'		,     [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]		,     )
def 		A_(     A						:		Optional[Any]		,     A						:		str		,     A						:		Optional[Any]):
							with patch('datasets.utils.py_utils._single_map_nested') as mock_single_map_nested, patch(
							    'datasets.parallel.parallel.Pool') as mock_multiprocessing_pool:
														UpperCamelCase   			=     {f'''{i}''': i for i in range(A)}
														UpperCamelCase   			=     map_nested(lambda A: x + 10		,     A		,     num_proc=A		,     parallel_min_length=16)
														if expected_num_proc == 1:
																					assert mock_single_map_nested.called
																					assert not mock_multiprocessing_pool.called
														else:
																					assert not mock_single_map_nested.called
																					assert mock_multiprocessing_pool.called
																					assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc


class 			SCREAMING_SNAKE_CASE__      (     snake_case_):







				@require_tf
				def 						UpperCAmelCase_    (       self     )->		List[str]:







											'''simple docstring'''






											import tensorflow as tf
											from tensorflow.keras import layers

											UpperCamelCase   			=     layers.Dense(2     )

											def gen_random_output():
																		UpperCamelCase   			=     tf.random.uniform((1, 3)     )
																		return model(A_     ).numpy()

											with temp_seed(42						,				set_tensorflow=A_     ):
																		UpperCamelCase   			=     gen_random_output()
											with temp_seed(42						,				set_tensorflow=A_     ):
																		UpperCamelCase   			=     gen_random_output()
											UpperCamelCase   			=     gen_random_output()

											np.testing.assert_equal(A_						,				A_     )
											self.assertGreater(np.abs(outa - outa     ).sum()						,				0     )







				@require_torch
				def 						UpperCAmelCase_    (       self     )->		int:







											'''simple docstring'''






											import torch

											def gen_random_output():
																		UpperCamelCase   			=     torch.nn.Linear(3						,				2     )
																		UpperCamelCase   			=     torch.rand(1						,				3     )
																		return model(A_     ).detach().numpy()

											with temp_seed(42						,				set_pytorch=A_     ):
																		UpperCamelCase   			=     gen_random_output()
											with temp_seed(42						,				set_pytorch=A_     ):
																		UpperCamelCase   			=     gen_random_output()
											UpperCamelCase   			=     gen_random_output()

											np.testing.assert_equal(A_						,				A_     )
											self.assertGreater(np.abs(outa - outa     ).sum()						,				0     )







				def 						UpperCAmelCase_    (       self     )->		Union[str, Any]:







											'''simple docstring'''






											def gen_random_output():
																		return np.random.rand(1						,				3     )

											with temp_seed(42     ):
																		UpperCamelCase   			=     gen_random_output()
											with temp_seed(42     ):
																		UpperCamelCase   			=     gen_random_output()
											UpperCamelCase   			=     gen_random_output()

											np.testing.assert_equal(A_						,				A_     )
											self.assertGreater(np.abs(outa - outa     ).sum()						,				0     )







@pytest.mark.parametrize('input_data'		,     [{}])
def 		A_(     A						:		Any):
							UpperCamelCase   			=     NestedDataStructure(A).data
							assert output_data == input_data







@pytest.mark.parametrize(
    'data, expected_output'		,     [
        ({}, []),
        ([], []),
        ('foo', ['foo']),
        (['foo', 'bar'], ['foo', 'bar']),
        ([['foo', 'bar']], ['foo', 'bar']),
        ([[['foo'], ['bar']]], ['foo', 'bar']),
        ([[['foo'], 'bar']], ['foo', 'bar']),
        ({'a': 1, 'b': 2}, [1, 2]),
        ({'a': [1, 2], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, [4]]}, [1, 2, 3, 4]),
        ({'a': {'1': 1}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': [2]}, [1, 2]),
    ]		,     )
def 		A_(     A						:		Optional[Any]		,     A						:		Optional[int]):
							UpperCamelCase   			=     NestedDataStructure(A).flatten()
							assert output == expected_output







def 		A_(     ):
							UpperCamelCase   			=     A(x=1		,     y='foobar')
							UpperCamelCase   			=     {'x': 1, 'y': 'foobar'}
							assert asdict(A) == expected_output

							UpperCamelCase   			=     {'a': {'b': A(x=10		,     y='foo')}, 'c': [A(x=20		,     y='bar')]}
							UpperCamelCase   			=     {'a': {'b': {'x': 10, 'y': 'foo'}}, 'c': [{'x': 20, 'y': 'bar'}]}
							assert asdict(A) == expected_output

							with pytest.raises(A):
														asdict([1, A(x=10		,     y='foo')])







def 		A_(     A						:		str):
							return text.split()







def 		A_(     A						:		Optional[int]):
							yield (time.time(), content)
							time.sleep(2)
							yield (time.time(), content)







def 		A_(     ):
							with Pool(2) as pool:
														UpperCamelCase   			=     list(iflatmap_unordered(A		,     _split_text		,     kwargs_iterable=[{'text': 'hello there'}] * 10))
														assert out.count('hello') == 10
														assert out.count('there') == 10
														assert len(A) == 20

							# check multiprocess from pathos (uses dill for pickling)
							with multiprocess.Pool(2) as pool:
														UpperCamelCase   			=     list(iflatmap_unordered(A		,     _split_text		,     kwargs_iterable=[{'text': 'hello there'}] * 10))
														assert out.count('hello') == 10
														assert out.count('there') == 10
														assert len(A) == 20

							# check that we get items as fast as possible
							with Pool(2) as pool:
														UpperCamelCase   			=     []
														for yield_time, content in iflatmap_unordered(
														    A		,     _aseconds_generator_of_aitems_with_timing		,     kwargs_iterable=[{'content': 'a'}, {'content': 'b'}]):
																					assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
																					out.append(A)
														assert out.count('a') == 2
														assert out.count('b') == 2
														assert len(A) == 4






'''simple docstring'''
import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
						import tensorflow as tf

						from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
						from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

lowerCAmelCase					:    Dict           = logging.get_logger(__name__)


class 			SCREAMING_SNAKE_CASE__      (     enum.Enum):
				lowerCAmelCase_     			=							0
				lowerCAmelCase_     			=							1


@add_end_docstrings(snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""generated"""







				def __init__(       self						,				*A_						,				**A_     )->		Optional[int]:







											'''simple docstring'''






											super().__init__(*A_						,				**A_     )

											self.check_model_type(
											    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
											    if self.framework == 'tf'
											    else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING     )







				def 						UpperCAmelCase_    (       self						,				A_=None						,				A_=None						,				A_=None						,				A_=None						,				A_=None						,				A_=None						,				**A_						,				)->		Optional[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     {}
											if truncation is not None:
																		UpperCamelCase   			=     truncation

											UpperCamelCase   			=     generate_kwargs

											UpperCamelCase   			=     {}
											if return_tensors is not None and return_type is None:
																		UpperCamelCase   			=     ReturnType.TENSORS if return_tensors else ReturnType.TEXT
											if return_type is not None:
																		UpperCamelCase   			=     return_type

											if clean_up_tokenization_spaces is not None:
																		UpperCamelCase   			=     clean_up_tokenization_spaces

											if stop_sequence is not None:
																		UpperCamelCase   			=     self.tokenizer.encode(A_						,				add_special_tokens=A_     )
																		if len(A_     ) > 1:
																									warnings.warn(
																									    'Stopping on a multiple token sequence is not yet supported on transformers. The first token of'
																									    ' the stop sequence will be used as the stop sequence string in the interim.'     )
																		UpperCamelCase   			=     stop_sequence_ids[0]

											return preprocess_params, forward_params, postprocess_params







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_     )->		Optional[int]:







											'''simple docstring'''






											return True







				def 						UpperCAmelCase_    (       self						,				*A_						,				A_     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     self.model.config.prefix if self.model.config.prefix is not None else ''
											if isinstance(args[0]						,				A_     ):
																		if self.tokenizer.pad_token_id is None:
																									raise ValueError('Please make sure that the tokenizer has a pad_token_id when using a batch input'     )
																		UpperCamelCase   			=     ([prefix + arg for arg in args[0]],)
																		UpperCamelCase   			=     True

											elif isinstance(args[0]						,				A_     ):
																		UpperCamelCase   			=     (prefix + args[0],)
																		UpperCamelCase   			=     False
											else:
																		raise ValueError(
																		    F''' `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`'''     )
											UpperCamelCase   			=     self.tokenizer(*A_						,				padding=A_						,				truncation=A_						,				return_tensors=self.framework     )
											# This is produced by tokenizers but is an invalid generate kwargs
											if "token_type_ids" in inputs:
																		del inputs["token_type_ids"]
											return inputs







				def __call__(       self						,				*A_						,				**A_     )->		Union[str, Any]:







											'''simple docstring'''






											UpperCamelCase   			=     super().__call__(*A_						,				**A_     )
											if (
											    isinstance(args[0]						,				A_     )
											    and all(isinstance(A_						,				A_     ) for el in args[0]     )
											    and all(len(A_     ) == 1 for res in result     )
											):
																		return [res[0] for res in result]
											return result







				def 						UpperCAmelCase_    (       self						,				A_						,				A_=TruncationStrategy.DO_NOT_TRUNCATE						,				**A_     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     self._parse_and_tokenize(A_						,				truncation=A_						,				**A_     )
											return inputs







				def 						UpperCAmelCase_    (       self						,				A_						,				**A_     )->		int:







											'''simple docstring'''






											if self.framework == "pt":
																		UpperCamelCase			,	UpperCamelCase   			=     model_inputs['input_ids'].shape
											elif self.framework == "tf":
																		UpperCamelCase			,	UpperCamelCase   			=     tf.shape(model_inputs['input_ids']     ).numpy()

											UpperCamelCase   			=     generate_kwargs.get('min_length'						,				self.model.config.min_length     )
											UpperCamelCase   			=     generate_kwargs.get('max_length'						,				self.model.config.max_length     )
											self.check_inputs(A_						,				generate_kwargs['min_length']						,				generate_kwargs['max_length']     )
											UpperCamelCase   			=     self.model.generate(**A_						,				**A_     )
											UpperCamelCase   			=     output_ids.shape[0]
											if self.framework == "pt":
																		UpperCamelCase   			=     output_ids.reshape(A_						,				out_b // in_b						,				*output_ids.shape[1:]     )
											elif self.framework == "tf":
																		UpperCamelCase   			=     tf.reshape(A_						,				(in_b, out_b // in_b, *output_ids.shape[1:])     )
											return {"output_ids": output_ids}







				def 						UpperCAmelCase_    (       self						,				A_						,				A_=ReturnType.TEXT						,				A_=False     )->		Optional[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     []
											for output_ids in model_outputs["output_ids"][0]:
																		if return_type == ReturnType.TENSORS:
																									UpperCamelCase   			=     {F'''{self.return_name}_token_ids''': output_ids}
																		elif return_type == ReturnType.TEXT:
																									UpperCamelCase   			=     {
																									    F'''{self.return_name}_text''': self.tokenizer.decode(
																									        A_						,				skip_special_tokens=A_						,				clean_up_tokenization_spaces=A_						,				)
																									}
																		records.append(A_     )
											return records


@add_end_docstrings(snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""summary"""







				def __call__(       self						,				*A_						,				**A_     )->		Optional[int]:







											'''simple docstring'''






											return super().__call__(*A_						,				**A_     )







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_     )->		bool:







											'''simple docstring'''






											if max_length < min_length:
																		logger.warning(F'''Your min_length={min_length} must be inferior than your max_length={max_length}.'''     )

											if input_length < max_length:
																		logger.warning(
																		    F'''Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is '''
																		    'a summarization task, where outputs shorter than the input are typically wanted, you might '
																		    F'''consider decreasing max_length manually, e.g. summarizer(\'...\', max_length={input_length//2})'''     )


@add_end_docstrings(snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""translation"""







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_     )->		List[Any]:







											'''simple docstring'''






											if input_length > 0.9 * max_length:
																		logger.warning(
																		    F'''Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider '''
																		    'increasing your max_length manually, e.g. translator(\'...\', max_length=400)'     )
											return True







				def 						UpperCAmelCase_    (       self						,				*A_						,				A_=TruncationStrategy.DO_NOT_TRUNCATE						,				A_=None						,				A_=None     )->		Dict:







											'''simple docstring'''






											if getattr(self.tokenizer						,				'_build_translation_inputs'						,				A_     ):
																		return self.tokenizer._build_translation_inputs(
																		    *A_						,				return_tensors=self.framework						,				truncation=A_						,				src_lang=A_						,				tgt_lang=A_     )
											else:
																		return super()._parse_and_tokenize(*A_						,				truncation=A_     )







				def 						UpperCAmelCase_    (       self						,				A_=None						,				A_=None						,				**A_     )->		str:







											'''simple docstring'''






											UpperCamelCase			,	UpperCamelCase			,	UpperCamelCase   			=     super()._sanitize_parameters(**A_     )
											if src_lang is not None:
																		UpperCamelCase   			=     src_lang
											if tgt_lang is not None:
																		UpperCamelCase   			=     tgt_lang
											if src_lang is None and tgt_lang is None:
																		# Backward compatibility, direct arguments use is preferred.
																		UpperCamelCase   			=     kwargs.get('task'						,				self.task     )
																		UpperCamelCase   			=     task.split('_'     )
																		if task and len(A_     ) == 4:
																									# translation, XX, to YY
																									UpperCamelCase   			=     items[1]
																									UpperCamelCase   			=     items[3]
											return preprocess_params, forward_params, postprocess_params







				def __call__(       self						,				*A_						,				**A_     )->		Any:







											'''simple docstring'''






											return super().__call__(*A_						,				**A_     )




