


import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


snake_case__				:	List[str]					       =					logging.getLogger(__name__)





@dataclass(frozen=a__						)
class   SCREAMING_SNAKE_CASE_			:




       '''simple docstring'''



       _a   							=							42
       _a   							=							42
       _a   							=							None
       _a   							=							None
       _a   							=							None







@dataclass(frozen=a__						)
class   SCREAMING_SNAKE_CASE_			:




       '''simple docstring'''



       _a   							=							42
       _a   							=							None
       _a   							=							None
       _a   							=							None
       _a   							=							None


if is_torch_available():
      import torch
      from torch.utils.data import Dataset





      class   SCREAMING_SNAKE_CASE_			(a__						):




             '''simple docstring'''



             _a   							=							42



             def __init__(						self				:				Optional[Any]		,				__a				:				str		,				__a				:				PreTrainedTokenizer		,				__a				:				str		,				__a				:				Optional[int] = None		,				__a				:				Optional[Any]=False		,				__a				:				bool = False		,				)					->Dict:
                lowerCamelCase_						:   Any		= hans_processors[task]()

                lowerCamelCase_						:   Optional[int]		= os.path.join(
                    __a		,				"""cached_{}_{}_{}_{}""".format(
                        """dev""" if evaluate else """train"""		,				tokenizer.__class__.__name__		,				str(__a							)		,				__a		,				)		,				)
                lowerCamelCase_						:   List[str]		= processor.get_labels()
                if tokenizer.__class__ in (
                    RobertaTokenizer,
                    RobertaTokenizerFast,
                    XLMRobertaTokenizer,
                    BartTokenizer,
                    BartTokenizerFast,
                ):
                   # HACK(label indices are swapped in RoBERTa pretrained model)
                   lowerCamelCase_,	lowerCamelCase_						:   Tuple		= label_list[2], label_list[1]
                lowerCamelCase_						:   List[Any]		= label_list

                # Make sure only the first process in distributed training processes the dataset,
                # and the others will use the cache.
                lowerCamelCase_						:   List[Any]		= cached_features_file + """.lock"""
                with FileLock(__a							):
                   if os.path.exists(__a							) and not overwrite_cache:
                      logger.info(F'''Loading features from cached file {cached_features_file}'''							)
                      lowerCamelCase_						:   List[str]		= torch.load(__a							)
                   else:
                      logger.info(F'''Creating features from dataset file at {data_dir}'''							)

                      lowerCamelCase_						:   List[str]		= (
                          processor.get_dev_examples(__a							) if evaluate else processor.get_train_examples(__a							)
                      )

                      logger.info("""Training examples: %s"""		,				len(__a							)							)
                      lowerCamelCase_						:   str		= hans_convert_examples_to_features(__a		,				__a		,				__a		,				__a							)
                      logger.info("""Saving features into cached file %s"""		,				__a							)
                      torch.save(self.features		,				__a							)



             def __len__(						self				:				Dict							)					->Tuple:
                return len(self.features							)



             def __getitem__(						self				:				Tuple		,				__a				:				Dict							)					->InputFeatures:
                return self.features[i]




             def 				_lowerCAmelCase							(						self				:				Tuple							)					->Tuple:
                return self.label_list


if is_tf_available():
      import tensorflow as tf





      class   SCREAMING_SNAKE_CASE_			:




             '''simple docstring'''



             _a   							=							42



             def __init__(						self				:				List[Any]		,				__a				:				str		,				__a				:				PreTrainedTokenizer		,				__a				:				str		,				__a				:				Optional[int] = 128		,				__a				:				Optional[int]=False		,				__a				:				bool = False		,				)					->int:
                lowerCamelCase_						:   Tuple		= hans_processors[task]()
                lowerCamelCase_						:   str		= processor.get_labels()
                if tokenizer.__class__ in (
                    RobertaTokenizer,
                    RobertaTokenizerFast,
                    XLMRobertaTokenizer,
                    BartTokenizer,
                    BartTokenizerFast,
                ):
                   # HACK(label indices are swapped in RoBERTa pretrained model)
                   lowerCamelCase_,	lowerCamelCase_						:   Any		= label_list[2], label_list[1]
                lowerCamelCase_						:   Optional[int]		= label_list

                lowerCamelCase_						:   Any		= processor.get_dev_examples(__a							) if evaluate else processor.get_train_examples(__a							)
                lowerCamelCase_						:   Dict		= hans_convert_examples_to_features(__a		,				__a		,				__a		,				__a							)

                def gen():
                   for ex_index, ex in tqdm.tqdm(enumerate(self.features							)		,				desc="""convert examples to features"""							):
                      if ex_index % 10_000 == 0:
                         logger.info("""Writing example %d of %d""" % (ex_index, len(__a							))							)

                      yield (
                          {
                              "example_id": 0,
                              "input_ids": ex.input_ids,
                              "attention_mask": ex.attention_mask,
                              "token_type_ids": ex.token_type_ids,
                          },
                          ex.label,
                      )

                lowerCamelCase_						:   Union[str, Any]		= tf.data.Dataset.from_generator(
                    __a		,				(
                        {
                            """example_id""": tf.intaa,
                            """input_ids""": tf.intaa,
                            """attention_mask""": tf.intaa,
                            """token_type_ids""": tf.intaa,
                        },
                        tf.intaa,
                    )		,				(
                        {
                            """example_id""": tf.TensorShape([]							),
                            """input_ids""": tf.TensorShape([None, None]							),
                            """attention_mask""": tf.TensorShape([None, None]							),
                            """token_type_ids""": tf.TensorShape([None, None]							),
                        },
                        tf.TensorShape([]							),
                    )		,				)



             def 				_lowerCAmelCase							(						self				:				Dict							)					->Tuple:
                return self.dataset



             def __len__(						self				:				Tuple							)					->Dict:
                return len(self.features							)



             def __getitem__(						self				:				Union[str, Any]		,				__a				:				Optional[Any]							)					->InputFeatures:
                return self.features[i]




             def 				_lowerCAmelCase							(						self				:				Union[str, Any]							)					->Tuple:
                return self.label_list





class   SCREAMING_SNAKE_CASE_			(a__						):




       '''simple docstring'''



       def 				_lowerCAmelCase							(						self				:				Any		,				__a				:				Optional[Any]							)					->Dict:
          return self._create_examples(self._read_tsv(os.path.join(__a		,				"""heuristics_train_set.txt"""							)							)		,				"""train"""							)



       def 				_lowerCAmelCase							(						self				:				Union[str, Any]		,				__a				:				Union[str, Any]							)					->Union[str, Any]:
          return self._create_examples(self._read_tsv(os.path.join(__a		,				"""heuristics_evaluation_set.txt"""							)							)		,				"""dev"""							)



       def 				_lowerCAmelCase							(						self				:				int							)					->str:
          return ["contradiction", "entailment", "neutral"]




       def 				_lowerCAmelCase							(						self				:				Union[str, Any]		,				__a				:				Any		,				__a				:				Tuple							)					->List[Any]:
          lowerCamelCase_						:   Optional[int]		= []
          for i, line in enumerate(__a							):
             if i == 0:
                continue
             lowerCamelCase_						:   int		= """%s-%s""" % (set_type, line[0])
             lowerCamelCase_						:   int		= line[5]
             lowerCamelCase_						:   Tuple		= line[6]
             lowerCamelCase_						:   Any		= line[7][2:] if line[7].startswith("""ex"""							) else line[7]
             lowerCamelCase_						:   Optional[Any]		= line[0]
             examples.append(InputExample(guid=__a		,				text_a=__a		,				text_b=__a		,				label=__a		,				pairID=__a							)							)
          return examples







def    __lowerCamelCase   (      A__						: List[InputExample]						,						A__						: List[str]						,						A__						: int						,						A__						: PreTrainedTokenizer						,						)     ->    Optional[int]:
   lowerCamelCase_						:   List[str]		= {label: i for i, label in enumerate(A__					)}

   lowerCamelCase_						:   Optional[int]		= []
   for ex_index, example in tqdm.tqdm(enumerate(A__					)						,						desc="""convert examples to features"""					):
      if ex_index % 1_0000 == 0:
         logger.info("""Writing example %d""" % (ex_index)					)

      lowerCamelCase_						:   Union[str, Any]		= tokenizer(
          example.text_a						,						example.text_b						,						add_special_tokens=A__						,						max_length=A__						,						padding="""max_length"""						,						truncation=A__						,						return_overflowing_tokens=A__						,						)

      lowerCamelCase_						:   Dict		= label_map[example.label] if example.label in label_map else 0

      lowerCamelCase_						:   Dict		= int(example.pairID					)

      features.append(InputFeatures(**A__						,						label=A__						,						pairID=A__					)					)

   for i, example in enumerate(examples[:5]					):
      logger.info("""*** Example ***"""					)
      logger.info(f'''guid: {example}'''					)
      logger.info(f'''features: {features[i]}'''					)

   return features


snake_case__				:	Dict					       =					{
    'hans': 3,
}



snake_case__				:	List[str]					       =					{
    'hans': HansProcessor,
}









import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType





class   SCREAMING_SNAKE_CASE_			(a__						):




       '''simple docstring'''



       _a   							=							["image_processor", "tokenizer"]
       _a   							=							"LayoutLMv2ImageProcessor"
       _a   							=							("LayoutXLMTokenizer", "LayoutXLMTokenizerFast")



       def __init__(						self				:				Any		,				__a				:				str=None		,				__a				:				List[Any]=None		,				**__a				:				Optional[int]							)					->int:
          if "feature_extractor" in kwargs:
             warnings.warn(
                 """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
                 """ instead."""		,				__a		,				)
             lowerCamelCase_						:   str		= kwargs.pop("""feature_extractor"""							)

          lowerCamelCase_						:   List[Any]		= image_processor if image_processor is not None else feature_extractor
          if image_processor is None:
             raise ValueError("""You need to specify an `image_processor`."""							)
          if tokenizer is None:
             raise ValueError("""You need to specify a `tokenizer`."""							)

          super().__init__(__a		,				__a							)



       def __call__(						self				:				List[str]		,				__a				:				List[Any]		,				__a				:				Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None		,				__a				:				Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None		,				__a				:				Union[List[List[int]], List[List[List[int]]]] = None		,				__a				:				Optional[Union[List[int], List[List[int]]]] = None		,				__a				:				bool = True		,				__a				:				Union[bool, str, PaddingStrategy] = False		,				__a				:				Union[bool, str, TruncationStrategy] = None		,				__a				:				Optional[int] = None		,				__a				:				int = 0		,				__a				:				Optional[int] = None		,				__a				:				Optional[bool] = None		,				__a				:				Optional[bool] = None		,				__a				:				bool = False		,				__a				:				bool = False		,				__a				:				bool = False		,				__a				:				bool = False		,				__a				:				bool = True		,				__a				:				Optional[Union[str, TensorType]] = None		,				**__a				:				Any		,				)					->BatchEncoding:
          # verify input
          if self.image_processor.apply_ocr and (boxes is not None):
             raise ValueError(
                 """You cannot provide bounding boxes """
                 """if you initialized the image processor with apply_ocr set to True."""							)

          if self.image_processor.apply_ocr and (word_labels is not None):
             raise ValueError(
                 """You cannot provide word labels if you initialized the image processor with apply_ocr set to True."""							)

          if return_overflowing_tokens is True and return_offsets_mapping is False:
             raise ValueError("""You cannot return overflowing tokens without returning the offsets mapping."""							)

          # first, apply the image processor
          lowerCamelCase_						:   Tuple		= self.image_processor(images=__a		,				return_tensors=__a							)

          # second, apply the tokenizer
          if text is not None and self.image_processor.apply_ocr and text_pair is None:
             if isinstance(__a		,				__a							):
                lowerCamelCase_						:   Optional[Any]		= [text]  # add batch dimension (as the image processor always adds a batch dimension)
             lowerCamelCase_						:   Dict		= features["""words"""]

          lowerCamelCase_						:   int		= self.tokenizer(
              text=text if text is not None else features["""words"""]		,				text_pair=text_pair if text_pair is not None else None		,				boxes=boxes if boxes is not None else features["""boxes"""]		,				word_labels=__a		,				add_special_tokens=__a		,				padding=__a		,				truncation=__a		,				max_length=__a		,				stride=__a		,				pad_to_multiple_of=__a		,				return_token_type_ids=__a		,				return_attention_mask=__a		,				return_overflowing_tokens=__a		,				return_special_tokens_mask=__a		,				return_offsets_mapping=__a		,				return_length=__a		,				verbose=__a		,				return_tensors=__a		,				**__a		,				)

          # add pixel values
          lowerCamelCase_						:   Any		= features.pop("""pixel_values"""							)
          if return_overflowing_tokens is True:
             lowerCamelCase_						:   Dict		= self.get_overflowing_images(__a		,				encoded_inputs["""overflow_to_sample_mapping"""]							)
          lowerCamelCase_						:   Optional[int]		= images

          return encoded_inputs



       def 				_lowerCAmelCase							(						self				:				Optional[Any]		,				__a				:				str		,				__a				:				Union[str, Any]							)					->List[str]:
          # in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
          lowerCamelCase_						:   Dict		= []
          for sample_idx in overflow_to_sample_mapping:
             images_with_overflow.append(images[sample_idx]							)

          if len(__a							) != len(__a							):
             raise ValueError(
                 """Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"""
                 F''' {len(__a							)} and {len(__a							)}'''							)

          return images_with_overflow



       def 				_lowerCAmelCase							(						self				:				Dict		,				*__a				:				Dict		,				**__a				:				int							)					->int:
          return self.tokenizer.batch_decode(*__a		,				**__a							)



       def 				_lowerCAmelCase							(						self				:				List[Any]		,				*__a				:				List[str]		,				**__a				:				int							)					->List[Any]:
          return self.tokenizer.decode(*__a		,				**__a							)



       @property
       def 				_lowerCAmelCase							(						self				:				int							)					->List[str]:
          return ["input_ids", "bbox", "attention_mask", "image"]



       @property
       def 				_lowerCAmelCase							(						self				:				int							)					->str:
          warnings.warn(
              """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""		,				__a		,				)
          return self.image_processor_class




       @property
       def 				_lowerCAmelCase							(						self				:				int							)					->List[Any]:
          warnings.warn(
              """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""		,				__a		,				)
          return self.image_processor





