


'''simple docstring'''






import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch






def 	_A				(lowerCAmelCase__  :Optional[Any]		)   ->  Optional[int]:  # picklable for multiprocessing




     '''simple docstring'''



     return x.sum()






def 	_A				(lowerCAmelCase__  :Dict		)   ->  Optional[Any]:  # picklable for multiprocessing




     '''simple docstring'''



     return i + 1




@dataclass
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	42





class  a     (     _SCREAMING_SNAKE_CASE       ):
 def 							__UpperCAmelCase		( self						)					->							Union[str, Any]:
      _a			      =							{}
      _a			      =							[]
      _a			      =							1
      _a			      =							[1, 2]
      _a			      =							{'a': 1, 'b': 2}
      _a			      =							{'a': [1, 2], 'b': [3, 4]}
      _a			      =							{'a': {'1': 1}, 'b': 2}
      _a			      =							{'a': 1, 'b': 2, 'c': 3, 'd': 4}
      _a			      =							{}
      _a			      =							[]
      _a			      =							2
      _a			      =							[2, 3]
      _a			      =							{'a': 2, 'b': 3}
      _a			      =							{'a': [2, 3], 'b': [4, 5]}
      _a			      =							{'a': {'1': 2}, 'b': 3}
      _a			      =							{'a': 2, 'b': 3, 'c': 4, 'd': 5}
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__						)      ,				__magic_name__						)

      _a			      =							2
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)

      _a			      =							{'a': np.eye(2						), 'b': np.zeros(3						), 'c': np.ones(2						)}
      _a			      =							{'a': 2, 'b': 0, 'c': 2}
      _a			      =							{
          'a': np.eye(2						).astype(__magic_name__						),
          'b': np.zeros(3						).astype(__magic_name__						),
          'c': np.ones(2						).astype(__magic_name__						),
      }
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				map_numpy=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(
          {k: v.tolist() for k, v in map_nested(__magic_name__      ,				__magic_name__      ,				map_numpy=__magic_name__						).items()}      ,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}      ,				)
      self.assertEqual(map_nested(__magic_name__      ,				__magic_name__      ,				map_numpy=__magic_name__      ,				num_proc=__magic_name__						)      ,				__magic_name__						)
      self.assertEqual(
          {k: v.tolist() for k, v in map_nested(__magic_name__      ,				__magic_name__      ,				map_numpy=__magic_name__      ,				num_proc=__magic_name__						).items()}      ,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}      ,				)
      with self.assertRaises(__magic_name__						):  # can't pickle a local lambda
           map_nested(lambda __magic_name__						: x + 1      ,				__magic_name__      ,				num_proc=__magic_name__						)
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      _a			      =							{'a': 1, 'b': 2}
      _a			      =							{'a': 3, 'b': 4}
      _a			      =							{'a': 5, 'b': 6}
      _a			      =							sorted([('a', (1, 3, 5)), ('b', (2, 4, 6))]						)
      self.assertEqual(sorted(zip_dict(__magic_name__      ,				__magic_name__      ,				__magic_name__						)						)      ,				__magic_name__						)




 def 							__UpperCAmelCase		( self						)					->							Optional[int]:




      class  a     :
       _lowerCAmelCase 					=	"""bar"""





      _a			      =							Foo()
      self.assertEqual(foo.my_attr      ,				'bar'						)
      with temporary_assignment(__magic_name__      ,				'my_attr'      ,				'BAR'						):
           self.assertEqual(foo.my_attr      ,				'BAR'						)
      self.assertEqual(foo.my_attr      ,				'bar'						)






@pytest.mark.parametrize(
    'iterable_length, num_proc, expected_num_proc'				,						[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]				,						)
def 	_A				(lowerCAmelCase__  :Optional[int]				,						lowerCAmelCase__  :int				,						lowerCAmelCase__  :Union[str, Any]		)   ->  Dict:




     '''simple docstring'''



     with patch('datasets.utils.py_utils._single_map_nested'		) as mock_single_map_nested, patch(
         'datasets.parallel.parallel.Pool'		) as mock_multiprocessing_pool:
          _a			      =							{f'{i}': i for i in range(lowerCAmelCase__		)}
          _a			      =							map_nested(lambda lowerCAmelCase__		: x + 10				,						lowerCAmelCase__				,						num_proc=lowerCAmelCase__				,						parallel_min_length=16		)
          if expected_num_proc == 1:
               assert mock_single_map_nested.called
               assert not mock_multiprocessing_pool.called
          else:
               assert not mock_single_map_nested.called
               assert mock_multiprocessing_pool.called
               assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc




class  a     (     _SCREAMING_SNAKE_CASE       ):
 @require_tf
 def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
      import tensorflow as tf
      from tensorflow.keras import layers

      _a			      =							layers.Dense(2						)

      def gen_random_output():
           _a			      =							tf.random.uniform((1, 3)						)
           return model(__magic_name__						).numpy()

      with temp_seed(42      ,				set_tensorflow=__magic_name__						):
           _a			      =							gen_random_output()
      with temp_seed(42      ,				set_tensorflow=__magic_name__						):
           _a			      =							gen_random_output()
      _a			      =							gen_random_output()

      np.testing.assert_equal(__magic_name__      ,				__magic_name__						)
      self.assertGreater(np.abs(outa - outa						).sum()      ,				0						)
 @require_torch
 def 							__UpperCAmelCase		( self						)					->							Union[str, Any]:
      import torch

      def gen_random_output():
           _a			      =							torch.nn.Linear(3      ,				2						)
           _a			      =							torch.rand(1      ,				3						)
           return model(__magic_name__						).detach().numpy()

      with temp_seed(42      ,				set_pytorch=__magic_name__						):
           _a			      =							gen_random_output()
      with temp_seed(42      ,				set_pytorch=__magic_name__						):
           _a			      =							gen_random_output()
      _a			      =							gen_random_output()

      np.testing.assert_equal(__magic_name__      ,				__magic_name__						)
      self.assertGreater(np.abs(outa - outa						).sum()      ,				0						)




 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      def gen_random_output():
           return np.random.rand(1      ,				3						)

      with temp_seed(42						):
           _a			      =							gen_random_output()
      with temp_seed(42						):
           _a			      =							gen_random_output()
      _a			      =							gen_random_output()

      np.testing.assert_equal(__magic_name__      ,				__magic_name__						)
      self.assertGreater(np.abs(outa - outa						).sum()      ,				0						)






@pytest.mark.parametrize('input_data'				,						[{}]		)
def 	_A				(lowerCAmelCase__  :Optional[Any]		)   ->  Optional[int]:




     '''simple docstring'''



     _a			      =							NestedDataStructure(lowerCAmelCase__		).data
     assert output_data == input_data






@pytest.mark.parametrize(
    'data, expected_output'				,						[
        ({}, []),
        ([], []),
        ('foo', ['foo']),
        (['foo', 'bar'], ['foo', 'bar']),
        ([['foo', 'bar']], ['foo', 'bar']),
        ([[['foo'], ['bar']]], ['foo', 'bar']),
        ([[['foo'], 'bar']], ['foo', 'bar']),
        ({'a': 1, 'b': 2}, [1, 2]),
        ({'a': [1, 2], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, [4]]}, [1, 2, 3, 4]),
        ({'a': {'1': 1}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': [2]}, [1, 2]),
    ]				,						)
def 	_A				(lowerCAmelCase__  :int				,						lowerCAmelCase__  :List[str]		)   ->  Tuple:




     '''simple docstring'''



     _a			      =							NestedDataStructure(lowerCAmelCase__		).flatten()
     assert output == expected_output






def 	_A				()   ->  int:




     '''simple docstring'''



     _a			      =							A(x=1				,						y='foobar'		)
     _a			      =							{'x': 1, 'y': 'foobar'}
     assert asdict(lowerCAmelCase__		) == expected_output

     _a			      =							{'a': {'b': A(x=10				,						y='foo'		)}, 'c': [A(x=20				,						y='bar'		)]}
     _a			      =							{'a': {'b': {'x': 10, 'y': 'foo'}}, 'c': [{'x': 20, 'y': 'bar'}]}
     assert asdict(lowerCAmelCase__		) == expected_output

     with pytest.raises(lowerCAmelCase__		):
          asdict([1, A(x=10				,						y='foo'		)]		)






def 	_A				(lowerCAmelCase__  :str		)   ->  Any:




     '''simple docstring'''



     return text.split()






def 	_A				(lowerCAmelCase__  :Dict		)   ->  Optional[int]:




     '''simple docstring'''



     yield (time.time(), content)
     time.sleep(2		)
     yield (time.time(), content)






def 	_A				()   ->  Tuple:




     '''simple docstring'''



     with Pool(2		) as pool:
          _a			      =							list(iflatmap_unordered(lowerCAmelCase__				,						_split_text				,						kwargs_iterable=[{'text': 'hello there'}] * 10		)		)
          assert out.count('hello'		) == 10
          assert out.count('there'		) == 10
          assert len(lowerCAmelCase__		) == 20

     # check multiprocess from pathos (uses dill for pickling)
     with multiprocess.Pool(2		) as pool:
          _a			      =							list(iflatmap_unordered(lowerCAmelCase__				,						_split_text				,						kwargs_iterable=[{'text': 'hello there'}] * 10		)		)
          assert out.count('hello'		) == 10
          assert out.count('there'		) == 10
          assert len(lowerCAmelCase__		) == 20

     # check that we get items as fast as possible
     with Pool(2		) as pool:
          _a			      =							[]
          for yield_time, content in iflatmap_unordered(
              lowerCAmelCase__				,						_aseconds_generator_of_aitems_with_timing				,						kwargs_iterable=[{'content': 'a'}, {'content': 'b'}]		):
               assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
               out.append(lowerCAmelCase__		)
          assert out.count('a'		) == 2
          assert out.count('b'		) == 2
          assert len(lowerCAmelCase__		) == 4





'''simple docstring'''






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


a_					:    Optional[Any]          =    logging.getLogger(__name__)




@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None





@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset




 class  a     (     _SCREAMING_SNAKE_CASE       ):
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = None      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							Optional[int]:
       _a			      =							hans_processors[task]()

       _a			      =							os.path.join(
           __magic_name__      ,				'cached_{}_{}_{}_{}'.format(
               'dev' if evaluate else 'train'      ,				tokenizer.__class__.__name__      ,				str(__magic_name__						)      ,				__magic_name__      ,				)      ,				)
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       # Make sure only the first process in distributed training processes the dataset,
       # and the others will use the cache.
       _a			      =							cached_features_file + '.lock'
       with FileLock(__magic_name__						):
            if os.path.exists(__magic_name__						) and not overwrite_cache:
                 logger.info(f'Loading features from cached file {cached_features_file}'						)
                 _a			      =							torch.load(__magic_name__						)
            else:
                 logger.info(f'Creating features from dataset file at {data_dir}'						)

                 _a			      =							(
                     processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
                 )

                 logger.info('Training examples: %s'      ,				len(__magic_name__						)						)
                 _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)
                 logger.info('Saving features into cached file %s'      ,				__magic_name__						)
                 torch.save(self.features      ,				__magic_name__						)
  def __len__( self						)					->							List[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
       return self.label_list


if is_tf_available():
 import tensorflow as tf




 class  a     :
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = 1_28      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							str:
       _a			      =							hans_processors[task]()
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       _a			      =							processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
       _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)

       def gen():
            for ex_index, ex in tqdm.tqdm(enumerate(self.features						)      ,				desc='convert examples to features'						):
                 if ex_index % 1_00_00 == 0:
                      logger.info('Writing example %d of %d' % (ex_index, len(__magic_name__						))						)

                 yield (
                     {
                         "example_id": 0,
                         "input_ids": ex.input_ids,
                         "attention_mask": ex.attention_mask,
                         "token_type_ids": ex.token_type_ids,
                     },
                     ex.label,
                 )

       _a			      =							tf.data.Dataset.from_generator(
           __magic_name__      ,				(
               {
                   'example_id': tf.intaa,
                   'input_ids': tf.intaa,
                   'attention_mask': tf.intaa,
                   'token_type_ids': tf.intaa,
               },
               tf.intaa,
           )      ,				(
               {
                   'example_id': tf.TensorShape([]						),
                   'input_ids': tf.TensorShape([None, None]						),
                   'attention_mask': tf.TensorShape([None, None]						),
                   'token_type_ids': tf.TensorShape([None, None]						),
               },
               tf.TensorShape([]						),
           )      ,				)
  def 							__UpperCAmelCase		( self						)					->							Tuple:
       return self.dataset
  def __len__( self						)					->							Optional[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							List[Any]:
       return self.label_list




class  a     (     _SCREAMING_SNAKE_CASE       ):
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							Tuple:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_train_set.txt'						)						)      ,				'train'						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							List[Any]:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_evaluation_set.txt'						)						)      ,				'dev'						)
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      return ["contradiction", "entailment", "neutral"]




 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							Optional[Any]:
      _a			      =							[]
      for i, line in enumerate(__magic_name__						):
           if i == 0:
                continue
           _a			      =							'%s-%s' % (set_type, line[0])
           _a			      =							line[5]
           _a			      =							line[6]
           _a			      =							line[7][2:] if line[7].startswith('ex'						) else line[7]
           _a			      =							line[0]
           examples.append(InputExample(guid=__magic_name__      ,				text_a=__magic_name__      ,				text_b=__magic_name__      ,				label=__magic_name__      ,				pairID=__magic_name__						)						)
      return examples






def 	_A				(lowerCAmelCase__  :List[InputExample]				,						lowerCAmelCase__  :List[str]				,						lowerCAmelCase__  :int				,						lowerCAmelCase__  :PreTrainedTokenizer				,						)   ->  Tuple:




     '''simple docstring'''



     _a			      =							{label: i for i, label in enumerate(lowerCAmelCase__		)}

     _a			      =							[]
     for ex_index, example in tqdm.tqdm(enumerate(lowerCAmelCase__		)				,						desc='convert examples to features'		):
          if ex_index % 1_00_00 == 0:
               logger.info('Writing example %d' % (ex_index)		)

          _a			      =							tokenizer(
              example.text_a				,						example.text_b				,						add_special_tokens=lowerCAmelCase__				,						max_length=lowerCAmelCase__				,						padding='max_length'				,						truncation=lowerCAmelCase__				,						return_overflowing_tokens=lowerCAmelCase__				,						)

          _a			      =							label_map[example.label] if example.label in label_map else 0

          _a			      =							int(example.pairID		)

          features.append(InputFeatures(**lowerCAmelCase__				,						label=lowerCAmelCase__				,						pairID=lowerCAmelCase__		)		)

     for i, example in enumerate(examples[:5]		):
          logger.info('*** Example ***'		)
          logger.info(f'guid: {example}'		)
          logger.info(f'features: {features[i]}'		)

     return features


a_					:    Optional[int]          =    {
    "hans": 3,
}






a_					:    Optional[Any]          =    {
    "hans": HansProcessor,
}

