
"""simple docstring"""

import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


UpperCamelCase     :    Dict       =			logging.getLogger(__name__)

@dataclass(frozen=__SCREAMING_SNAKE_CASE      )
class __lowerCAmelCase		:
     lowercase						       =						42
     lowercase						       =						42
     lowercase						       =						None
     lowercase						       =						None
     lowercase						       =						None


@dataclass(frozen=__SCREAMING_SNAKE_CASE      )
class __lowerCAmelCase		:
     lowercase						       =						42
     lowercase						       =						None
     lowercase						       =						None
     lowercase						       =						None
     lowercase						       =						None


if is_torch_available():
      import torch
      from torch.utils.data import Dataset

      class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ):
           lowercase						       =						42





           def __init__(     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase=False     ,					__UpperCAmelCase = False     ,					):

            '''simple docstring'''




            __UpperCamelCase      =	hans_processors[task]()

            __UpperCamelCase      =	os.path.join(
                __UpperCAmelCase     ,					'cached_{}_{}_{}_{}'.format(
                    'dev' if evaluate else 'train'     ,					tokenizer.__class__.__name__     ,					str(__UpperCAmelCase				)     ,					__UpperCAmelCase     ,					)     ,					)
            __UpperCamelCase      =	processor.get_labels()
            if tokenizer.__class__ in (
                RobertaTokenizer,
                RobertaTokenizerFast,
                XLMRobertaTokenizer,
                BartTokenizer,
                BartTokenizerFast,
            ):
             # HACK(label indices are swapped in RoBERTa pretrained model)
             __UpperCamelCase  ,     __UpperCamelCase      =	label_list[2], label_list[1]
            __UpperCamelCase      =	label_list

            # Make sure only the first process in distributed training processes the dataset,
            # and the others will use the cache.
            __UpperCamelCase      =	cached_features_file + '.lock'
            with FileLock(__UpperCAmelCase				):
             if os.path.exists(__UpperCAmelCase				) and not overwrite_cache:
              logger.info(F'Loading features from cached file {cached_features_file}'				)
              __UpperCamelCase      =	torch.load(__UpperCAmelCase				)
             else:
              logger.info(F'Creating features from dataset file at {data_dir}'				)

              __UpperCamelCase      =	(
                  processor.get_dev_examples(__UpperCAmelCase				) if evaluate else processor.get_train_examples(__UpperCAmelCase				)
              )

              logger.info('Training examples: %s'     ,					len(__UpperCAmelCase				)				)
              __UpperCamelCase      =	hans_convert_examples_to_features(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase				)
              logger.info('Saving features into cached file %s'     ,					__UpperCAmelCase				)
              torch.save(self.features     ,					__UpperCAmelCase				)





           def __len__(     self				):

            '''simple docstring'''




            return len(self.features				)





           def __getitem__(     self     ,					__UpperCAmelCase				):

            '''simple docstring'''




            return self.features[i]







           def 						UpperCAmelCase (     self				):

            '''simple docstring'''




            return self.label_list


if is_tf_available():
      import tensorflow as tf

      class __lowerCAmelCase		:
           lowercase						       =						42





           def __init__(     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase = 128     ,					__UpperCAmelCase=False     ,					__UpperCAmelCase = False     ,					):

            '''simple docstring'''




            __UpperCamelCase      =	hans_processors[task]()
            __UpperCamelCase      =	processor.get_labels()
            if tokenizer.__class__ in (
                RobertaTokenizer,
                RobertaTokenizerFast,
                XLMRobertaTokenizer,
                BartTokenizer,
                BartTokenizerFast,
            ):
             # HACK(label indices are swapped in RoBERTa pretrained model)
             __UpperCamelCase  ,     __UpperCamelCase      =	label_list[2], label_list[1]
            __UpperCamelCase      =	label_list

            __UpperCamelCase      =	processor.get_dev_examples(__UpperCAmelCase				) if evaluate else processor.get_train_examples(__UpperCAmelCase				)
            __UpperCamelCase      =	hans_convert_examples_to_features(__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase				)

            def gen():
             for ex_index, ex in tqdm.tqdm(enumerate(self.features				)     ,					desc='convert examples to features'				):
              if ex_index % 1_0000 == 0:
               logger.info('Writing example %d of %d' % (ex_index, len(__UpperCAmelCase				))				)

              yield (
                  {
                      "example_id": 0,
                      "input_ids": ex.input_ids,
                      "attention_mask": ex.attention_mask,
                      "token_type_ids": ex.token_type_ids,
                  },
                  ex.label,
              )

            __UpperCamelCase      =	tf.data.Dataset.from_generator(
                __UpperCAmelCase     ,					(
                    {
                        'example_id': tf.intaa,
                        'input_ids': tf.intaa,
                        'attention_mask': tf.intaa,
                        'token_type_ids': tf.intaa,
                    },
                    tf.intaa,
                )     ,					(
                    {
                        'example_id': tf.TensorShape([]				),
                        'input_ids': tf.TensorShape([None, None]				),
                        'attention_mask': tf.TensorShape([None, None]				),
                        'token_type_ids': tf.TensorShape([None, None]				),
                    },
                    tf.TensorShape([]				),
                )     ,					)





           def 						UpperCAmelCase (     self				):

            '''simple docstring'''




            return self.dataset





           def __len__(     self				):

            '''simple docstring'''




            return len(self.features				)





           def __getitem__(     self     ,					__UpperCAmelCase				):

            '''simple docstring'''




            return self.features[i]







           def 						UpperCAmelCase (     self				):

            '''simple docstring'''




            return self.label_list

class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ):





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase				):

      '''simple docstring'''




      return self._create_examples(self._read_tsv(os.path.join(__UpperCAmelCase     ,					'heuristics_train_set.txt'				)				)     ,					'train'				)





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase				):

      '''simple docstring'''




      return self._create_examples(self._read_tsv(os.path.join(__UpperCAmelCase     ,					'heuristics_evaluation_set.txt'				)				)     ,					'dev'				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      return ["contradiction", "entailment", "neutral"]







     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase				):

      '''simple docstring'''




      __UpperCamelCase      =	[]
      for i, line in enumerate(__UpperCAmelCase				):
       if i == 0:
        continue
       __UpperCamelCase      =	'%s-%s' % (set_type, line[0])
       __UpperCamelCase      =	line[5]
       __UpperCamelCase      =	line[6]
       __UpperCamelCase      =	line[7][2:] if line[7].startswith('ex'				) else line[7]
       __UpperCamelCase      =	line[0]
       examples.append(InputExample(guid=__UpperCAmelCase     ,					text_a=__UpperCAmelCase     ,					text_b=__UpperCAmelCase     ,					label=__UpperCAmelCase     ,					pairID=__UpperCAmelCase				)				)
      return examples






def 							A						(    snake_case				:List[InputExample]    , snake_case				:List[str]    , snake_case				:int    , snake_case				:PreTrainedTokenizer    , )				->     int:
 __UpperCamelCase      =	{label: i for i, label in enumerate(snake_case		)}

 __UpperCamelCase      =	[]
 for ex_index, example in tqdm.tqdm(enumerate(snake_case		)    , desc='convert examples to features'		):
  if ex_index % 1_0_0_0_0 == 0:
   logger.info('Writing example %d' % (ex_index)		)

  __UpperCamelCase      =	tokenizer(
      example.text_a    , example.text_b    , add_special_tokens=snake_case    , max_length=snake_case    , padding='max_length'    , truncation=snake_case    , return_overflowing_tokens=snake_case    , )

  __UpperCamelCase      =	label_map[example.label] if example.label in label_map else 0

  __UpperCamelCase      =	int(example.pairID		)

  features.append(InputFeatures(**snake_case    , label=snake_case    , pairID=snake_case		)		)

 for i, example in enumerate(examples[:5]		):
  logger.info('*** Example ***'		)
  logger.info(f'guid: {example}'		)
  logger.info(f'features: {features[i]}'		)

 return features


UpperCamelCase     :    Any       =			{
    "hans": 3,
}





UpperCamelCase     :    Union[str, Any]       =			{
    "hans": HansProcessor,
}





"""simple docstring"""

import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ):





     def __init__(     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = False     ,					__UpperCAmelCase = False     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      super().__init__(
          __UpperCAmelCase     ,					split=__UpperCAmelCase     ,					features=__UpperCAmelCase     ,					cache_dir=__UpperCAmelCase     ,					keep_in_memory=__UpperCAmelCase     ,					streaming=__UpperCAmelCase     ,					num_proc=__UpperCAmelCase     ,					**__UpperCAmelCase     ,					)
      __UpperCamelCase      =	field
      __UpperCamelCase      =	path_or_paths if isinstance(__UpperCAmelCase     ,					__UpperCAmelCase				) else {self.split: path_or_paths}
      __UpperCamelCase      =	Json(
          cache_dir=__UpperCAmelCase     ,					data_files=__UpperCAmelCase     ,					features=__UpperCAmelCase     ,					field=__UpperCAmelCase     ,					**__UpperCAmelCase     ,					)







     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      if self.streaming:
       __UpperCamelCase      =	self.builder.as_streaming_dataset(split=self.split				)
      # Build regular (map-style) dataset
      else:
       __UpperCamelCase      =	None
       __UpperCamelCase      =	None
       __UpperCamelCase      =	None
       __UpperCamelCase      =	None

       self.builder.download_and_prepare(
           download_config=__UpperCAmelCase     ,					download_mode=__UpperCAmelCase     ,					verification_mode=__UpperCAmelCase     ,					base_path=__UpperCAmelCase     ,					num_proc=self.num_proc     ,					)
       __UpperCamelCase      =	self.builder.as_dataset(
           split=self.split     ,					verification_mode=__UpperCAmelCase     ,					in_memory=self.keep_in_memory				)
      return dataset


class __lowerCAmelCase		:





     def __init__(     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase = None     ,					__UpperCAmelCase = None     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      if num_proc is not None and num_proc <= 0:
       raise ValueError(F'num_proc {num_proc} must be an integer > 0.'				)

      __UpperCamelCase      =	dataset
      __UpperCamelCase      =	path_or_buf
      __UpperCamelCase      =	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
      __UpperCamelCase      =	num_proc
      __UpperCamelCase      =	'utf-8'
      __UpperCamelCase      =	to_json_kwargs





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.to_json_kwargs.pop('path_or_buf'     ,					__UpperCAmelCase				)
      __UpperCamelCase      =	self.to_json_kwargs.pop('orient'     ,					'records'				)
      __UpperCamelCase      =	self.to_json_kwargs.pop('lines'     ,					True if orient == 'records' else False				)
      __UpperCamelCase      =	self.to_json_kwargs.pop('index'     ,					False if orient in ['split', 'table'] else True				)
      __UpperCamelCase      =	self.to_json_kwargs.pop('compression'     ,					__UpperCAmelCase				)

      if compression not in [None, "infer", "gzip", "bz2", "xz"]:
       raise NotImplementedError(F'`datasets` currently does not support {compression} compression'				)

      if isinstance(self.path_or_buf     ,					(str, bytes, os.PathLike)				):
       with fsspec.open(self.path_or_buf     ,					'wb'     ,					compression=__UpperCAmelCase				) as buffer:
        __UpperCamelCase      =	self._write(file_obj=__UpperCAmelCase     ,					orient=__UpperCAmelCase     ,					lines=__UpperCAmelCase     ,					index=__UpperCAmelCase     ,					**self.to_json_kwargs				)
      else:
       if compression:
        raise NotImplementedError(
            F'The compression parameter is not supported when writing to a buffer, but compression={compression}'
            ' was passed. Please provide a local path instead.'				)
       __UpperCamelCase      =	self._write(
           file_obj=self.path_or_buf     ,					orient=__UpperCAmelCase     ,					lines=__UpperCAmelCase     ,					index=__UpperCAmelCase     ,					**self.to_json_kwargs				)
      return written





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase				):

      '''simple docstring'''




      __UpperCamelCase  ,     __UpperCamelCase  ,     __UpperCamelCase  ,     __UpperCamelCase  ,     __UpperCamelCase      =	args

      __UpperCamelCase      =	query_table(
          table=self.dataset.data     ,					key=slice(__UpperCAmelCase     ,					offset + self.batch_size				)     ,					indices=self.dataset._indices     ,					)
      __UpperCamelCase      =	batch.to_pandas().to_json(
          path_or_buf=__UpperCAmelCase     ,					orient=__UpperCAmelCase     ,					lines=__UpperCAmelCase     ,					index=__UpperCAmelCase     ,					**__UpperCAmelCase				)
      if not json_str.endswith('\n'				):
       json_str += "\n"
      return json_str.encode(self.encoding				)







     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					__UpperCAmelCase     ,					**__UpperCAmelCase     ,					):

      '''simple docstring'''




      __UpperCamelCase      =	0

      if self.num_proc is None or self.num_proc == 1:
       for offset in logging.tqdm(
           range(0     ,					len(self.dataset				)     ,					self.batch_size				)     ,					unit='ba'     ,					disable=not logging.is_progress_bar_enabled()     ,					desc='Creating json from Arrow format'     ,					):
        __UpperCamelCase      =	self._batch_json((offset, orient, lines, index, to_json_kwargs)				)
        written += file_obj.write(__UpperCAmelCase				)
      else:
       __UpperCamelCase  ,     __UpperCamelCase      =	len(self.dataset				), self.batch_size
       with multiprocessing.Pool(self.num_proc				) as pool:
        for json_str in logging.tqdm(
            pool.imap(
                self._batch_json     ,					[(offset, orient, lines, index, to_json_kwargs) for offset in range(0     ,					__UpperCAmelCase     ,					__UpperCAmelCase				)]     ,					)     ,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size     ,					unit='ba'     ,					disable=not logging.is_progress_bar_enabled()     ,					desc='Creating json from Arrow format'     ,					):
         written += file_obj.write(__UpperCAmelCase				)

      return written



