






import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    LiltForQuestionAnswering,
							    LiltForSequenceClassification,
							    LiltForTokenClassification,
							    LiltModel,
							)
							from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST


class   __SCREAMING_SNAKE_CASE:
							def __init__(     self:      Any       ,	UpperCamelCase:      Tuple       ,	UpperCamelCase:      str=13       ,	UpperCamelCase:      int=7       ,	UpperCamelCase:      Optional[int]=True       ,	UpperCamelCase:      Tuple=True       ,	UpperCamelCase:      List[Any]=True       ,	UpperCamelCase:      Dict=True       ,	UpperCamelCase:      Any=99       ,	UpperCamelCase:      int=24       ,	UpperCamelCase:      List[Any]=2       ,	UpperCamelCase:      Any=6       ,	UpperCamelCase:      Union[str, Any]=37       ,	UpperCamelCase:      int="gelu"       ,	UpperCamelCase:      Optional[Any]=0.1       ,	UpperCamelCase:      Optional[Any]=0.1       ,	UpperCamelCase:      Union[str, Any]=5_12       ,	UpperCamelCase:      List[Any]=16       ,	UpperCamelCase:      str=2       ,	UpperCamelCase:      Any=0.02       ,	UpperCamelCase:      int=3       ,	UpperCamelCase:      str=None       ,	UpperCamelCase:      str=10_00       ,	)		->		Any:
											snake_case__									=     parent
											snake_case__									=     batch_size
											snake_case__									=     seq_length
											snake_case__									=     is_training
											snake_case__									=     use_input_mask
											snake_case__									=     use_token_type_ids
											snake_case__									=     use_labels
											snake_case__									=     vocab_size
											snake_case__									=     hidden_size
											snake_case__									=     num_hidden_layers
											snake_case__									=     num_attention_heads
											snake_case__									=     intermediate_size
											snake_case__									=     hidden_act
											snake_case__									=     hidden_dropout_prob
											snake_case__									=     attention_probs_dropout_prob
											snake_case__									=     max_position_embeddings
											snake_case__									=     type_vocab_size
											snake_case__									=     type_sequence_label_size
											snake_case__									=     initializer_range
											snake_case__									=     num_labels
											snake_case__									=     scope
											snake_case__									=     range_bbox
							def lowerCAmelCase_ (     self:      Optional[Any]						)		->		Dict:
											snake_case__									=     ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size						)

											snake_case__									=     ids_tensor([self.batch_size, self.seq_length, 4]       ,	self.range_bbox						)
											# Ensure that bbox is legal
											for i in range(bbox.shape[0]						):
															for j in range(bbox.shape[1]						):
																			if bbox[i, j, 3] < bbox[i, j, 1]:
																							snake_case__									=     bbox[i, j, 3]
																							snake_case__									=     bbox[i, j, 1]
																							snake_case__									=     t
																			if bbox[i, j, 2] < bbox[i, j, 0]:
																							snake_case__									=     bbox[i, j, 2]
																							snake_case__									=     bbox[i, j, 0]
																							snake_case__									=     t

											snake_case__									=     None
											if self.use_input_mask:
															snake_case__									=     ids_tensor([self.batch_size, self.seq_length]       ,	vocab_size=2						)

											snake_case__									=     None
											if self.use_token_type_ids:
															snake_case__									=     ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size						)

											snake_case__									=     None
											snake_case__									=     None
											if self.use_labels:
															snake_case__									=     ids_tensor([self.batch_size]       ,	self.type_sequence_label_size						)
															snake_case__									=     ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels						)

											snake_case__									=     self.get_config()

											return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels
							def lowerCAmelCase_ (     self:      int						)		->		Optional[Any]:
											return LiltConfig(
											    vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_size=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout_prob=self.hidden_dropout_prob       ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	initializer_range=self.initializer_range       ,	)
							def lowerCAmelCase_ (     self:      Optional[int]       ,	UpperCamelCase:      Optional[Any]       ,	UpperCamelCase:      str       ,	UpperCamelCase:      Any       ,	UpperCamelCase:      int       ,	UpperCamelCase:      Union[str, Any]       ,	UpperCamelCase:      Optional[Any]       ,	UpperCamelCase:      Tuple       ,	)		->		Dict:
											snake_case__									=     LiltModel(config=UpperCamelCase						)
											model.to(UpperCamelCase						)
											model.eval()
											snake_case__									=     model(UpperCamelCase       ,	bbox=UpperCamelCase       ,	attention_mask=UpperCamelCase       ,	token_type_ids=UpperCamelCase						)
											snake_case__									=     model(UpperCamelCase       ,	bbox=UpperCamelCase       ,	token_type_ids=UpperCamelCase						)
											snake_case__									=     model(UpperCamelCase       ,	bbox=UpperCamelCase						)

											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)						)
											self.parent.assertEqual(result.pooler_output.shape       ,	(self.batch_size, self.hidden_size)						)
							def lowerCAmelCase_ (     self:      Optional[Any]       ,	UpperCamelCase:      Optional[Any]       ,	UpperCamelCase:      Optional[Any]       ,	UpperCamelCase:      List[str]       ,	UpperCamelCase:      str       ,	UpperCamelCase:      Union[str, Any]       ,	UpperCamelCase:      Any       ,	UpperCamelCase:      Optional[Any]       ,	)		->		Tuple:
											snake_case__									=     self.num_labels
											snake_case__									=     LiltForTokenClassification(config=UpperCamelCase						)
											model.to(UpperCamelCase						)
											model.eval()
											snake_case__									=     model(
											    UpperCamelCase       ,	bbox=UpperCamelCase       ,	attention_mask=UpperCamelCase       ,	token_type_ids=UpperCamelCase       ,	labels=UpperCamelCase						)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)						)
							def lowerCAmelCase_ (     self:      Optional[int]       ,	UpperCamelCase:      List[str]       ,	UpperCamelCase:      Dict       ,	UpperCamelCase:      Tuple       ,	UpperCamelCase:      Any       ,	UpperCamelCase:      List[Any]       ,	UpperCamelCase:      Union[str, Any]       ,	UpperCamelCase:      List[Any]       ,	)		->		Any:
											snake_case__									=     LiltForQuestionAnswering(config=UpperCamelCase						)
											model.to(UpperCamelCase						)
											model.eval()
											snake_case__									=     model(
											    UpperCamelCase       ,	bbox=UpperCamelCase       ,	attention_mask=UpperCamelCase       ,	token_type_ids=UpperCamelCase       ,	start_positions=UpperCamelCase       ,	end_positions=UpperCamelCase       ,	)
											self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)						)
											self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)						)


							def lowerCAmelCase_ (     self:      List[str]						)		->		Optional[int]:
											snake_case__									=     self.prepare_config_and_inputs()
											(
											    (
											    snake_case__
											)							,						(
											    snake_case__
											)							,						(
											    snake_case__
											)							,						(
											    snake_case__
											)							,						(
											    snake_case__
											)							,						(
											    snake_case__
											)							,						(
											    snake_case__
											)							,						
											)									=     config_and_inputs
											snake_case__									=     {
											    'input_ids': input_ids,
											    'bbox': bbox,
											    'token_type_ids': token_type_ids,
											    'attention_mask': input_mask,
											}
											return config, inputs_dict


@require_torch
class   __SCREAMING_SNAKE_CASE(	a_    ,  a_    ,  a_    ,  unittest.TestCase	):
							_UpperCAmelCase     =       (
							    (
							        LiltModel,
							        LiltForSequenceClassification,
							        LiltForTokenClassification,
							        LiltForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							_UpperCAmelCase     =       (
							    {
							        "feature-extraction": LiltModel,
							        "question-answering": LiltForQuestionAnswering,
							        "text-classification": LiltForSequenceClassification,
							        "token-classification": LiltForTokenClassification,
							        "zero-shot": LiltForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       False
							def lowerCAmelCase_ (     self:      Dict       ,	UpperCamelCase:      Dict       ,	UpperCamelCase:      List[str]       ,	UpperCamelCase:      Any       ,	UpperCamelCase:      Union[str, Any]       ,	UpperCamelCase:      int						)		->		int:
											return True
							def lowerCAmelCase_ (     self:      Dict						)		->		List[Any]:
											snake_case__									=     LiltModelTester(self						)
											snake_case__									=     ConfigTester(self       ,	config_class=UpperCamelCase       ,	hidden_size=37						)
							def lowerCAmelCase_ (     self:      Optional[Any]						)		->		Tuple:
											self.config_tester.run_common_tests()
							def lowerCAmelCase_ (     self:      Union[str, Any]						)		->		str:
											snake_case__									=     self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Tuple						)		->		Dict:
											snake_case__									=     self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
															snake_case__									=     type
															self.model_tester.create_and_check_model(*UpperCamelCase						)
							def lowerCAmelCase_ (     self:      int						)		->		Dict:
											snake_case__									=     self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Dict						)		->		int:
											snake_case__									=     self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*UpperCamelCase						)


							@slow
							def lowerCAmelCase_ (     self:      int						)		->		Any:
											for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															snake_case__									=     LiltModel.from_pretrained(UpperCamelCase						)
															self.assertIsNotNone(UpperCamelCase						)


@require_torch
@slow
class   __SCREAMING_SNAKE_CASE(	unittest.TestCase	):
							def lowerCAmelCase_ (     self:      List[Any]						)		->		Dict:
											snake_case__									=     LiltModel.from_pretrained('SCUT-DLVCLab/lilt-roberta-en-base'						).to(UpperCamelCase						)

											snake_case__									=     torch.tensor([[1, 2]]       ,	device=UpperCamelCase						)
											snake_case__									=     torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]       ,	device=UpperCamelCase						)

											# forward pass
											with torch.no_grad():
															snake_case__									=     model(input_ids=UpperCamelCase       ,	bbox=UpperCamelCase						)

											snake_case__									=     torch.Size([1, 2, 7_68]						)
											snake_case__									=     torch.tensor(
											    [[-0.0_653, 0.0_950, -0.0_061], [-0.0_545, 0.0_926, -0.0_324]]       ,	device=UpperCamelCase       ,	)

											self.assertTrue(outputs.last_hidden_state.shape       ,	UpperCamelCase						)
											self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]       ,	UpperCamelCase       ,	atol=1e-3						)						)












import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


__UpperCamelCase					:		Any	  =						datasets.utils.logging.get_logger(__name__)

__UpperCamelCase					:		Union[str, Any]	  =						["""names""", """prefix"""]
__UpperCamelCase					:		List[str]	  =						["""warn_bad_lines""", """error_bad_lines""", """mangle_dupe_cols"""]
__UpperCamelCase					:		Dict	  =						["""encoding_errors""", """on_bad_lines"""]
__UpperCamelCase					:		Any	  =						["""date_format"""]


@dataclass
class   __SCREAMING_SNAKE_CASE(	datasets.BuilderConfig	):
							_UpperCAmelCase     =       ","
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       "infer"
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       "."
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       '"'
							_UpperCAmelCase     =       0
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       0
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       1_0_0_0_0
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       "strict"
							_UpperCAmelCase     =       "error"
							_UpperCAmelCase     =       None
							def lowerCAmelCase_ (     self:      Tuple						)		->		int:
											if self.delimiter is not None:
															snake_case__									=     self.delimiter
											if self.column_names is not None:
															snake_case__									=     self.column_names


							@property
							def lowerCAmelCase_ (     self:      Optional[int]						)		->		List[str]:
											snake_case__									=     {
											    'sep': self.sep,
											    'header': self.header,
											    'names': self.names,
											    'index_col': self.index_col,
											    'usecols': self.usecols,
											    'prefix': self.prefix,
											    'mangle_dupe_cols': self.mangle_dupe_cols,
											    'engine': self.engine,
											    'converters': self.converters,
											    'true_values': self.true_values,
											    'false_values': self.false_values,
											    'skipinitialspace': self.skipinitialspace,
											    'skiprows': self.skiprows,
											    'nrows': self.nrows,
											    'na_values': self.na_values,
											    'keep_default_na': self.keep_default_na,
											    'na_filter': self.na_filter,
											    'verbose': self.verbose,
											    'skip_blank_lines': self.skip_blank_lines,
											    'thousands': self.thousands,
											    'decimal': self.decimal,
											    'lineterminator': self.lineterminator,
											    'quotechar': self.quotechar,
											    'quoting': self.quoting,
											    'escapechar': self.escapechar,
											    'comment': self.comment,
											    'encoding': self.encoding,
											    'dialect': self.dialect,
											    'error_bad_lines': self.error_bad_lines,
											    'warn_bad_lines': self.warn_bad_lines,
											    'skipfooter': self.skipfooter,
											    'doublequote': self.doublequote,
											    'memory_map': self.memory_map,
											    'float_precision': self.float_precision,
											    'chunksize': self.chunksize,
											    'encoding_errors': self.encoding_errors,
											    'on_bad_lines': self.on_bad_lines,
											    'date_format': self.date_format,
											}

											# some kwargs must not be passed if they don't have a default value
											# some others are deprecated and we can also not pass them if they are the default value
											for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
															if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()       ,	UpperCamelCase						):
																			del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
											if not (datasets.config.PANDAS_VERSION.major >= 2):
															for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
																			del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
											if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
															for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
																			del pd_read_csv_kwargs[pd_read_csv_parameter]

											return pd_read_csv_kwargs


class   __SCREAMING_SNAKE_CASE(	datasets.ArrowBasedBuilder	):
							_UpperCAmelCase     =       CsvConfig
							def lowerCAmelCase_ (     self:      str						)		->		int:
											return datasets.DatasetInfo(features=self.config.features						)
							def lowerCAmelCase_ (     self:      Tuple       ,	UpperCamelCase:      Dict						)		->		int:
											if not self.config.data_files:
															raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''						)
											snake_case__									=     dl_manager.download_and_extract(self.config.data_files						)
											if isinstance(UpperCamelCase       ,	(str, list, tuple)						):
															snake_case__									=     data_files
															if isinstance(UpperCamelCase       ,	UpperCamelCase						):
																			snake_case__									=     [files]
															snake_case__									=     [dl_manager.iter_files(UpperCamelCase						) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={'files': files}						)]
											snake_case__									=     []
											for split_name, files in data_files.items():
															if isinstance(UpperCamelCase       ,	UpperCamelCase						):
																			snake_case__									=     [files]
															snake_case__									=     [dl_manager.iter_files(UpperCamelCase						) for file in files]
															splits.append(datasets.SplitGenerator(name=UpperCamelCase       ,	gen_kwargs={'files': files}						)						)
											return splits
							def lowerCAmelCase_ (     self:      int       ,	UpperCamelCase:      pa.Table						)		->		pa.Table:
											if self.config.features is not None:
															snake_case__									=     self.config.features.arrow_schema
															if all(not require_storage_cast(UpperCamelCase						) for feature in self.config.features.values()						):
																			# cheaper cast
																			snake_case__									=     pa.Table.from_arrays([pa_table[field.name] for field in schema]       ,	schema=UpperCamelCase						)
															else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			snake_case__									=     table_cast(UpperCamelCase       ,	UpperCamelCase						)
											return pa_table


							def lowerCAmelCase_ (     self:      str       ,	UpperCamelCase:      Tuple						)		->		Tuple:
											snake_case__									=     self.config.features.arrow_schema if self.config.features else None
											# dtype allows reading an int column as str
											snake_case__									=     (
											    {
											        name: dtype.to_pandas_dtype() if not require_storage_cast(UpperCamelCase						) else object
											        for name, dtype, feature in zip(schema.names       ,	schema.types       ,	self.config.features.values()						)
											    }
											    if schema is not None
											    else None
											)
											for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase						)						):
															snake_case__									=     pd.read_csv(UpperCamelCase       ,	iterator=UpperCamelCase       ,	dtype=UpperCamelCase       ,	**self.config.pd_read_csv_kwargs						)
															try:
																			for batch_idx, df in enumerate(UpperCamelCase						):
																							snake_case__									=     pa.Table.from_pandas(UpperCamelCase						)
																							# Uncomment for debugging (will print the Arrow table size and elements)
																							# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																							# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																							yield (file_idx, batch_idx), self._cast_table(UpperCamelCase						)
															except ValueError as e:
																			logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase						)}: {e}'''						)
																			raise




