





'''simple docstring'''



import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch




def  _lowerCAmelCase    (					lowerCamelCase_      :	Any					):  # picklable for multiprocessing
			return x.sum()




def  _lowerCAmelCase    (					lowerCamelCase_      :	Optional[Any]					):  # picklable for multiprocessing
			return i + 1






@dataclass
class 				__lowercase							:







	'''simple docstring'''

	a     :					int
	a     :					str






class 				__lowercase							(   snake_case__    ):







	'''simple docstring'''

	def     _UpperCAmelCase  (self			)				->				Optional[int]:







				'''simple docstring'''





				__lowercase					    =				{}
				__lowercase					    =				[]
				__lowercase					    =				1
				__lowercase					    =				[1, 2]
				__lowercase					    =				{'''a''': 1, '''b''': 2}
				__lowercase					    =				{'''a''': [1, 2], '''b''': [3, 4]}
				__lowercase					    =				{'''a''': {'''1''': 1}, '''b''': 2}
				__lowercase					    =				{'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
				__lowercase					    =				{}
				__lowercase					    =				[]
				__lowercase					    =				2
				__lowercase					    =				[2, 3]
				__lowercase					    =				{'''a''': 2, '''b''': 3}
				__lowercase					    =				{'''a''': [2, 3], '''b''': [4, 5]}
				__lowercase					    =				{'''a''': {'''1''': 2}, '''b''': 3}
				__lowercase					    =				{'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase			)  ,_UpperCAmelCase			)

				__lowercase					    =				2
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)

				__lowercase					    =				{'''a''': np.eye(2			), '''b''': np.zeros(3			), '''c''': np.ones(2			)}
				__lowercase					    =				{'''a''': 2, '''b''': 0, '''c''': 2}
				__lowercase					    =				{
				    '''a''': np.eye(2			).astype(_UpperCAmelCase			),
				    '''b''': np.zeros(3			).astype(_UpperCAmelCase			),
				    '''c''': np.ones(2			).astype(_UpperCAmelCase			),
				}
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,map_numpy=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(
				    {k: v.tolist() for k, v in map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,map_numpy=_UpperCAmelCase			).items()}  ,{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}  ,)
				self.assertEqual(map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,map_numpy=_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)  ,_UpperCAmelCase			)
				self.assertEqual(
				    {k: v.tolist() for k, v in map_nested(_UpperCAmelCase  ,_UpperCAmelCase  ,map_numpy=_UpperCAmelCase  ,num_proc=_UpperCAmelCase			).items()}  ,{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}  ,)
				with self.assertRaises(_UpperCAmelCase			):  # can't pickle a local lambda
							map_nested(lambda _lowerCamelCase			: x + 1  ,_UpperCAmelCase  ,num_proc=_UpperCAmelCase			)


	def     _UpperCAmelCase  (self			)				->				Tuple:







				'''simple docstring'''





				__lowercase					    =				{'''a''': 1, '''b''': 2}
				__lowercase					    =				{'''a''': 3, '''b''': 4}
				__lowercase					    =				{'''a''': 5, '''b''': 6}
				__lowercase					    =				sorted([('''a''', (1, 3, 5)), ('''b''', (2, 4, 6))]			)
				self.assertEqual(sorted(zip_dict(_UpperCAmelCase  ,_UpperCAmelCase  ,_UpperCAmelCase			)			)  ,_UpperCAmelCase			)


	def     _UpperCAmelCase  (self			)				->				str:







				'''simple docstring'''





				class 				__lowercase							:







					'''simple docstring'''

					a     :					Optional[int]           =							'''bar'''





				__lowercase					    =				Foo()
				self.assertEqual(foo.my_attr  ,'''bar'''			)
				with temporary_assignment(_UpperCAmelCase  ,'''my_attr'''  ,'''BAR'''			):
							self.assertEqual(foo.my_attr  ,'''BAR'''			)
				self.assertEqual(foo.my_attr  ,'''bar'''			)





@pytest.mark.parametrize(
    '''iterable_length, num_proc, expected_num_proc'''		,							[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (1_6, 1_6, 1_6),
        (1_6, 1_7, 1_6),
        (1_7, 1_6, 1_6),
    ]		,							)
def  _lowerCAmelCase    (					lowerCamelCase_      :	Union[str, Any]		,							lowerCamelCase_      :	Optional[Any]		,							lowerCamelCase_      :	Tuple					):
			with patch('''datasets.utils.py_utils._single_map_nested'''					) as mock_single_map_nested, patch(
			    '''datasets.parallel.parallel.Pool'''					) as mock_multiprocessing_pool:
						__lowercase					    =				{f"{i}": i for i in range(UpperCamelCase_					)}
						__lowercase					    =				map_nested(lambda lowerCamelCase_					: x + 1_0		,							UpperCamelCase_		,							num_proc=UpperCamelCase_		,							parallel_min_length=1_6					)
						if expected_num_proc == 1:
									assert mock_single_map_nested.called
									assert not mock_multiprocessing_pool.called
						else:
									assert not mock_single_map_nested.called
									assert mock_multiprocessing_pool.called
									assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc






class 				__lowercase							(   snake_case__    ):







	'''simple docstring'''

	@require_tf
	def     _UpperCAmelCase  (self			)				->				int:







				'''simple docstring'''





				import tensorflow as tf
				from tensorflow.keras import layers

				__lowercase					    =				layers.Dense(2			)

				def gen_random_output():
							__lowercase					    =				tf.random.uniform((1, 3)			)
							return model(_UpperCAmelCase			).numpy()

				with temp_seed(42  ,set_tensorflow=_UpperCAmelCase			):
							__lowercase					    =				gen_random_output()
				with temp_seed(42  ,set_tensorflow=_UpperCAmelCase			):
							__lowercase					    =				gen_random_output()
				__lowercase					    =				gen_random_output()

				np.testing.assert_equal(_UpperCAmelCase  ,_UpperCAmelCase			)
				self.assertGreater(np.abs(outa - outa			).sum()  ,0			)


	@require_torch
	def     _UpperCAmelCase  (self			)				->				Optional[int]:







				'''simple docstring'''





				import torch

				def gen_random_output():
							__lowercase					    =				torch.nn.Linear(3  ,2			)
							__lowercase					    =				torch.rand(1  ,3			)
							return model(_UpperCAmelCase			).detach().numpy()

				with temp_seed(42  ,set_pytorch=_UpperCAmelCase			):
							__lowercase					    =				gen_random_output()
				with temp_seed(42  ,set_pytorch=_UpperCAmelCase			):
							__lowercase					    =				gen_random_output()
				__lowercase					    =				gen_random_output()

				np.testing.assert_equal(_UpperCAmelCase  ,_UpperCAmelCase			)
				self.assertGreater(np.abs(outa - outa			).sum()  ,0			)


	def     _UpperCAmelCase  (self			)				->				int:







				'''simple docstring'''





				def gen_random_output():
							return np.random.rand(1  ,3			)

				with temp_seed(42			):
							__lowercase					    =				gen_random_output()
				with temp_seed(42			):
							__lowercase					    =				gen_random_output()
				__lowercase					    =				gen_random_output()

				np.testing.assert_equal(_UpperCAmelCase  ,_UpperCAmelCase			)
				self.assertGreater(np.abs(outa - outa			).sum()  ,0			)





@pytest.mark.parametrize('''input_data'''		,							[{}]					)
def  _lowerCAmelCase    (					lowerCamelCase_      :	Optional[int]					):
			__lowercase					    =				NestedDataStructure(UpperCamelCase_					).data
			assert output_data == input_data




@pytest.mark.parametrize(
    '''data, expected_output'''		,							[
        ({}, []),
        ([], []),
        ('''foo''', ['''foo''']),
        (['''foo''', '''bar'''], ['''foo''', '''bar''']),
        ([['''foo''', '''bar''']], ['''foo''', '''bar''']),
        ([[['''foo'''], ['''bar''']]], ['''foo''', '''bar''']),
        ([[['''foo'''], '''bar''']], ['''foo''', '''bar''']),
        ({'''a''': 1, '''b''': 2}, [1, 2]),
        ({'''a''': [1, 2], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, [4]]}, [1, 2, 3, 4]),
        ({'''a''': {'''1''': 1}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': [2]}, [1, 2]),
    ]		,							)
def  _lowerCAmelCase    (					lowerCamelCase_      :	Tuple		,							lowerCamelCase_      :	Optional[Any]					):
			__lowercase					    =				NestedDataStructure(UpperCamelCase_					).flatten()
			assert output == expected_output




def  _lowerCAmelCase    (					):
			__lowercase					    =				A(x=1		,							y='''foobar'''					)
			__lowercase					    =				{'''x''': 1, '''y''': '''foobar'''}
			assert asdict(UpperCamelCase_					) == expected_output

			__lowercase					    =				{'''a''': {'''b''': A(x=1_0		,							y='''foo'''					)}, '''c''': [A(x=2_0		,							y='''bar'''					)]}
			__lowercase					    =				{'''a''': {'''b''': {'''x''': 1_0, '''y''': '''foo'''}}, '''c''': [{'''x''': 2_0, '''y''': '''bar'''}]}
			assert asdict(UpperCamelCase_					) == expected_output

			with pytest.raises(UpperCamelCase_					):
						asdict([1, A(x=1_0		,							y='''foo'''					)]					)




def  _lowerCAmelCase    (					lowerCamelCase_      :	str					):
			return text.split()




def  _lowerCAmelCase    (					lowerCamelCase_      :	List[str]					):
			yield (time.time(), content)
			time.sleep(2					)
			yield (time.time(), content)




def  _lowerCAmelCase    (					):
			with Pool(2					) as pool:
						__lowercase					    =				list(iflatmap_unordered(UpperCamelCase_		,							_split_text		,							kwargs_iterable=[{'''text''': '''hello there'''}] * 1_0					)					)
						assert out.count('''hello'''					) == 1_0
						assert out.count('''there'''					) == 1_0
						assert len(UpperCamelCase_					) == 2_0

			# check multiprocess from pathos (uses dill for pickling)
			with multiprocess.Pool(2					) as pool:
						__lowercase					    =				list(iflatmap_unordered(UpperCamelCase_		,							_split_text		,							kwargs_iterable=[{'''text''': '''hello there'''}] * 1_0					)					)
						assert out.count('''hello'''					) == 1_0
						assert out.count('''there'''					) == 1_0
						assert len(UpperCamelCase_					) == 2_0

			# check that we get items as fast as possible
			with Pool(2					) as pool:
						__lowercase					    =				[]
						for yield_time, content in iflatmap_unordered(
						    UpperCamelCase_		,							_aseconds_generator_of_aitems_with_timing		,							kwargs_iterable=[{'''content''': '''a'''}, {'''content''': '''b'''}]					):
									assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
									out.append(UpperCamelCase_					)
						assert out.count('''a'''					) == 2
						assert out.count('''b'''					) == 2
						assert len(UpperCamelCase_					) == 4










'''simple docstring'''

import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__     :						int	=  get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')

@require_sentencepiece
@require_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: List[Any]									=      GPTSwaTokenizer
			a_	: int									=      False
			a_	: List[Any]									=      True
			a_	: Any									=      False






			def 						_lowercase  (     self						)							->							str:
					super().setUp()

					# We have a SentencePiece fixture for testing
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase					,   eos_token='<unk>'					,   bos_token='<unk>'					,   pad_token='<unk>'						)

					tokenizer.save_pretrained(self.tmpdirname						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__	     ='This is a test'
					snake_case__	     ='This is a test'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							str:
					snake_case__	     ='<s>'
					snake_case__	     =1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase						)					,   _UpperCAmelCase						)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =list(self.get_tokenizer().get_vocab().keys()						)

					self.assertEqual(vocab_keys[0]					,   '<unk>'						)
					self.assertEqual(vocab_keys[1]					,   '<s>'						)
					self.assertEqual(vocab_keys[-1]					,   'j'						)
					self.assertEqual(len(_UpperCAmelCase						)					,   2000						)






			def 						_lowercase  (     self						)							->							List[str]:
					self.assertEqual(self.get_tokenizer().vocab_size					,   2000						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase						)

					snake_case__	     =tokenizer.tokenize('This is a test'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁This', '▁is', '▁a', '▁t', 'est']						)

					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [465, 287, 265, 631, 842]						)

					snake_case__	     =tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
					# fmt: off
					self.assertListEqual(
					    _UpperCAmelCase					,   ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']					,   )
					# fmt: on

					snake_case__	     =tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]					,   )

					snake_case__	     =tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
					# fmt: off
					self.assertListEqual(
					    _UpperCAmelCase					,   ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']						)
					# fmt: on






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase						)
					snake_case__	     =['This is a test', 'I was born in 92000, and this is falsé.']
					snake_case__	     =[
					    [465, 287, 265, 631, 842],
					    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
					]

					# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
					for text, expected_ids in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							self.assertListEqual(tokenizer.encode_fast(_UpperCAmelCase						)					,   _UpperCAmelCase						)

					# Test that decode_fast returns the input text
					for text, token_ids in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							self.assertEqual(tokenizer.decode_fast(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			@slow
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =[
					    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
					    'Hey there, how are you doing this fine day?',
					    'This is a text with a trailing spaces followed by a dot     .',
					    'Häj sväjs lillebrör! =)',
					    'Det är inget fel på Mr. Cool',
					]

					# fmt: off
					snake_case__	     ={'input_ids': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
					# fmt: on
					self.tokenizer_integration_test_util(
					    expected_encoding=_UpperCAmelCase					,   model_name='AI-Sweden/gpt-sw3-126m'					,   sequences=_UpperCAmelCase					,   )




