





'''simple docstring'''



import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
	import torch

	from ..models.auto.modeling_auto import (
	    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
	    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
	)

if is_tf_available() and is_tensorflow_probability_available():
	import tensorflow as tf
	import tensorflow_probability as tfp

	from ..models.auto.modeling_tf_auto import (
	    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
	    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
	)






class 							__lowercase				(					lowerCAmelCase__  ):







	'''simple docstring'''

	def __call__(self   ,_lowerCamelCase=None   ,_lowerCamelCase=None   ,**_lowerCamelCase       )							->							str:







				'''simple docstring'''





				requires_backends(self   ,'''pandas'''       )
				import pandas as pd

				if table is None:
							raise ValueError('''Keyword argument `table` cannot be None.'''       )
				elif query is None:
							if isinstance(_lowerCamelCase   ,_lowerCamelCase       ) and table.get('''query'''       ) is not None and table.get('''table'''       ) is not None:
										__lowercase   =							[table]
							elif isinstance(_lowerCamelCase   ,_lowerCamelCase       ) and len(_lowerCamelCase       ) > 0:
										if not all(isinstance(_lowerCamelCase   ,_lowerCamelCase       ) for d in table       ):
													raise ValueError(
													    f"Keyword argument `table` should be a list of dict, but is {(type(_lowerCamelCase       ) for d in table)}"       )

										if table[0].get('''query'''       ) is not None and table[0].get('''table'''       ) is not None:
													__lowercase   =							table
										else:
													raise ValueError(
													    '''If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`'''
													    f" and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys."       )
							elif Dataset is not None and isinstance(_lowerCamelCase   ,_lowerCamelCase       ) or isinstance(_lowerCamelCase   ,types.GeneratorType       ):
										return table
							else:
										raise ValueError(
										    '''Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but '''
										    f"is {type(_lowerCamelCase       )})"       )
				else:
							__lowercase   =							[{'''table''': table, '''query''': query}]

				for tqa_pipeline_input in tqa_pipeline_inputs:
							if not isinstance(tqa_pipeline_input['''table''']   ,pd.DataFrame       ):
										if tqa_pipeline_input["table"] is None:
													raise ValueError('''Table cannot be None.'''       )

										__lowercase   =							pd.DataFrame(tqa_pipeline_input['''table''']       )

				return tqa_pipeline_inputs






@add_end_docstrings(lowerCAmelCase__  )
class 							__lowercase				(					lowerCAmelCase__  ):







	'''simple docstring'''

	a			: str      =				"table,query"


	def __init__(self   ,_lowerCamelCase=TableQuestionAnsweringArgumentHandler()   ,*_lowerCamelCase   ,**_lowerCamelCase       )							->							Optional[int]:







				'''simple docstring'''





				super().__init__(*_lowerCamelCase   ,**_lowerCamelCase       )
				__lowercase   =							args_parser

				self.check_model_type(
				    dict(
				        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
				        + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()       )
				    if self.framework == '''tf'''
				    else dict(
				        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()       )       )

				__lowercase   =							bool(getattr(self.model.config   ,'''aggregation_labels'''   ,_lowerCamelCase       )       ) and bool(
				    getattr(self.model.config   ,'''num_aggregation_labels'''   ,_lowerCamelCase       )       )
				__lowercase   =							'''tapas''' if hasattr(self.model.config   ,'''aggregation_labels'''       ) else None


	def   _UpperCAmelCase   (self   ,**_lowerCamelCase       )							->							str:







				'''simple docstring'''





				return self.model(**_lowerCamelCase       )


	def   _UpperCAmelCase   (self   ,**_lowerCamelCase       )							->							Union[str, Any]:







				'''simple docstring'''





				if self.framework == "pt":
							__lowercase   =							[]
							__lowercase   =							[]
							__lowercase   =							None
							__lowercase   =							inputs['''input_ids'''].shape[0]

							__lowercase   =							inputs['''input_ids'''].to(self.device       )
							__lowercase   =							inputs['''attention_mask'''].to(self.device       )
							__lowercase   =							inputs['''token_type_ids'''].to(self.device       )
							__lowercase   =							None

							for index in range(_lowerCamelCase       ):
										# If sequences have already been processed, the token type IDs will be created according to the previous
										# answer.
										if prev_answers is not None:
													__lowercase   =							token_type_ids_example[:, 3]  # shape (seq_len,)
													__lowercase   =							np.zeros_like(prev_labels_example.cpu().numpy()       )  # shape (seq_len,)

													__lowercase   =							token_type_ids[index]  # shape (seq_len, 7)
													for i in range(model_labels.shape[0]       ):
																__lowercase   =							token_type_ids_example[:, 0].tolist()[i]
																__lowercase   =							token_type_ids_example[:, 1].tolist()[i] - 1
																__lowercase   =							token_type_ids_example[:, 2].tolist()[i] - 1

																if row_id >= 0 and col_id >= 0 and segment_id == 1:
																			__lowercase   =							int(prev_answers[(col_id, row_id)]       )

													__lowercase   =							torch.from_numpy(_lowerCamelCase       ).type(torch.long       ).to(self.device       )

										__lowercase   =							input_ids[index]
										__lowercase   =							attention_mask[index]  # shape (seq_len,)
										__lowercase   =							token_type_ids[index]  # shape (seq_len, 7)
										__lowercase   =							self.model(
										    input_ids=input_ids_example.unsqueeze(0       )   ,attention_mask=attention_mask_example.unsqueeze(0       )   ,token_type_ids=token_type_ids_example.unsqueeze(0       )   ,)
										__lowercase   =							outputs.logits

										if self.aggregate:
													all_aggregations.append(outputs.logits_aggregation       )

										all_logits.append(_lowerCamelCase       )

										__lowercase   =							torch.distributions.Bernoulli(logits=_lowerCamelCase       )
										__lowercase   =							dist_per_token.probs * attention_mask_example.type(torch.floataa       ).to(
										    dist_per_token.probs.device       )

										__lowercase   =							collections.defaultdict(_lowerCamelCase       )
										for i, p in enumerate(probabilities.squeeze().tolist()       ):
													__lowercase   =							token_type_ids_example[:, 0].tolist()[i]
													__lowercase   =							token_type_ids_example[:, 1].tolist()[i] - 1
													__lowercase   =							token_type_ids_example[:, 2].tolist()[i] - 1
													if col >= 0 and row >= 0 and segment_id == 1:
																coords_to_probs[(col, row)].append(_lowerCamelCase       )

										__lowercase   =							{key: np.array(coords_to_probs[key]       ).mean() > 0.5 for key in coords_to_probs}

							__lowercase   =							torch.cat(tuple(_lowerCamelCase       )   ,0       )

							return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(_lowerCamelCase       )   ,0       ))
				else:
							__lowercase   =							[]
							__lowercase   =							[]
							__lowercase   =							None
							__lowercase   =							inputs['''input_ids'''].shape[0]

							__lowercase   =							inputs['''input_ids''']
							__lowercase   =							inputs['''attention_mask''']
							__lowercase   =							inputs['''token_type_ids'''].numpy()
							__lowercase   =							None

							for index in range(_lowerCamelCase       ):
										# If sequences have already been processed, the token type IDs will be created according to the previous
										# answer.
										if prev_answers is not None:
													__lowercase   =							token_type_ids_example[:, 3]  # shape (seq_len,)
													__lowercase   =							np.zeros_like(_lowerCamelCase   ,dtype=np.intaa       )  # shape (seq_len,)

													__lowercase   =							token_type_ids[index]  # shape (seq_len, 7)
													for i in range(model_labels.shape[0]       ):
																__lowercase   =							token_type_ids_example[:, 0].tolist()[i]
																__lowercase   =							token_type_ids_example[:, 1].tolist()[i] - 1
																__lowercase   =							token_type_ids_example[:, 2].tolist()[i] - 1

																if row_id >= 0 and col_id >= 0 and segment_id == 1:
																			__lowercase   =							int(prev_answers[(col_id, row_id)]       )

													__lowercase   =							model_labels

										__lowercase   =							input_ids[index]
										__lowercase   =							attention_mask[index]  # shape (seq_len,)
										__lowercase   =							token_type_ids[index]  # shape (seq_len, 7)
										__lowercase   =							self.model(
										    input_ids=np.expand_dims(_lowerCamelCase   ,axis=0       )   ,attention_mask=np.expand_dims(_lowerCamelCase   ,axis=0       )   ,token_type_ids=np.expand_dims(_lowerCamelCase   ,axis=0       )   ,)
										__lowercase   =							outputs.logits

										if self.aggregate:
													all_aggregations.append(outputs.logits_aggregation       )

										all_logits.append(_lowerCamelCase       )

										__lowercase   =							tfp.distributions.Bernoulli(logits=_lowerCamelCase       )
										__lowercase   =							dist_per_token.probs_parameter() * tf.cast(_lowerCamelCase   ,tf.floataa       )

										__lowercase   =							collections.defaultdict(_lowerCamelCase       )
										__lowercase   =							token_type_ids_example
										for i, p in enumerate(tf.squeeze(_lowerCamelCase       ).numpy().tolist()       ):
													__lowercase   =							token_type_ids_example[:, 0].tolist()[i]
													__lowercase   =							token_type_ids_example[:, 1].tolist()[i] - 1
													__lowercase   =							token_type_ids_example[:, 2].tolist()[i] - 1
													if col >= 0 and row >= 0 and segment_id == 1:
																coords_to_probs[(col, row)].append(_lowerCamelCase       )

										__lowercase   =							{key: np.array(coords_to_probs[key]       ).mean() > 0.5 for key in coords_to_probs}

							__lowercase   =							tf.concat(tuple(_lowerCamelCase       )   ,0       )

							return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(_lowerCamelCase       )   ,0       ))


	def __call__(self   ,*_lowerCamelCase   ,**_lowerCamelCase       )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase   =							self._args_parser(*_lowerCamelCase   ,**_lowerCamelCase       )

				__lowercase   =							super().__call__(_lowerCamelCase   ,**_lowerCamelCase       )
				if len(_lowerCamelCase       ) == 1:
							return results[0]
				return results


	def   _UpperCAmelCase   (self   ,_lowerCamelCase=None   ,_lowerCamelCase=None   ,_lowerCamelCase=None   ,**_lowerCamelCase       )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase   =							{}
				if padding is not None:
							__lowercase   =							padding
				if truncation is not None:
							__lowercase   =							truncation

				__lowercase   =							{}
				if sequential is not None:
							__lowercase   =							sequential
				return preprocess_params, forward_params, {}


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase=None   ,_lowerCamelCase=True   ,_lowerCamelCase=None       )							->							Tuple:







				'''simple docstring'''





				if truncation is None:
							if self.type == "tapas":
										__lowercase   =							'''drop_rows_to_fit'''
							else:
										__lowercase   =							'''do_not_truncate'''

				__lowercase  ,						__lowercase   =							pipeline_input['''table'''], pipeline_input['''query''']
				if table.empty:
							raise ValueError('''table is empty'''       )
				if query is None or query == "":
							raise ValueError('''query is empty'''       )
				__lowercase   =							self.tokenizer(_lowerCamelCase   ,_lowerCamelCase   ,return_tensors=self.framework   ,truncation=_lowerCamelCase   ,padding=_lowerCamelCase       )
				__lowercase   =							table
				return inputs


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase=False       )							->							Tuple:







				'''simple docstring'''





				__lowercase   =							model_inputs.pop('''table'''       )

				if self.type == "tapas":
							if sequential:
										__lowercase   =							self.sequential_inference(**_lowerCamelCase       )
							else:
										__lowercase   =							self.batch_inference(**_lowerCamelCase       )
				else:
							__lowercase   =							self.model.generate(**_lowerCamelCase       )
				__lowercase   =							{'''model_inputs''': model_inputs, '''table''': table, '''outputs''': outputs}
				return model_outputs


	def   _UpperCAmelCase   (self   ,_lowerCamelCase       )							->							int:







				'''simple docstring'''





				__lowercase   =							model_outputs['''model_inputs''']
				__lowercase   =							model_outputs['''table''']
				__lowercase   =							model_outputs['''outputs''']
				if self.type == "tapas":
							if self.aggregate:
										__lowercase  ,						__lowercase   =							outputs[:2]
										__lowercase   =							self.tokenizer.convert_logits_to_predictions(_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase       )
										__lowercase  ,						__lowercase   =							predictions
										__lowercase   =							{i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(_lowerCamelCase       )}

										__lowercase   =							self.model.config.no_aggregation_label_index
										__lowercase   =							{
										    i: aggregators[i] + ''' > ''' for i, pred in enumerate(_lowerCamelCase       ) if pred != no_agg_label_index
										}
							else:
										__lowercase   =							outputs[0]
										__lowercase   =							self.tokenizer.convert_logits_to_predictions(_lowerCamelCase   ,_lowerCamelCase       )
										__lowercase   =							predictions[0]
										__lowercase   =							{}
										__lowercase   =							{}
							__lowercase   =							[]
							for index, coordinates in enumerate(_lowerCamelCase       ):
										__lowercase   =							[table.iat[coordinate] for coordinate in coordinates]
										__lowercase   =							aggregators.get(_lowerCamelCase   ,''''''       )
										__lowercase   =							aggregators_prefix.get(_lowerCamelCase   ,''''''       )
										__lowercase   =							{
										    '''answer''': aggregator_prefix + ''', '''.join(_lowerCamelCase       ),
										    '''coordinates''': coordinates,
										    '''cells''': [table.iat[coordinate] for coordinate in coordinates],
										}
										if aggregator:
													__lowercase   =							aggregator

										answers.append(_lowerCamelCase       )
							if len(_lowerCamelCase       ) == 0:
										raise PipelineException('''Empty answer'''       )
				else:
							__lowercase   =							[{'''answer''': answer} for answer in self.tokenizer.batch_decode(_lowerCamelCase   ,skip_special_tokens=_lowerCamelCase       )]

				return answers if len(_lowerCamelCase       ) > 1 else answers[0]












'''simple docstring'''



import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
_SCREAMING_SNAKE_CASE    =			'''\
@InProceedings{huggingface:dataset,
title = {A great new dataset},
author={huggingface, Inc.
},
year={2020}
}
'''

# TODO: Add description of the dataset here
# You can copy an official description
_SCREAMING_SNAKE_CASE    =			'''\
This new dataset is designed to solve this great NLP task and is crafted with a lot of care.
'''

# TODO: Add a link to an official homepage for the dataset here
_SCREAMING_SNAKE_CASE    =			''''''

# TODO: Add the licence for the dataset here if you can find it
_SCREAMING_SNAKE_CASE    =			''''''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
_SCREAMING_SNAKE_CASE    =			{
    '''first_domain''': '''https://huggingface.co/great-new-dataset-first_domain.zip''',
    '''second_domain''': '''https://huggingface.co/great-new-dataset-second_domain.zip''',
}






class 							__lowercase				(					datasets.GeneratorBasedBuilder  ):







	'''simple docstring'''

	a			: Tuple      =				datasets.Version("1.1.0"  )

	# This is an example of a dataset with multiple configurations.
	# If you don't want/need to define several sub-sets in your dataset,
	# just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

	# If you need to make complex sub-parts in the datasets with configurable options
	# You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
	# BUILDER_CONFIG_CLASS = MyBuilderConfig

	# You will be able to load one or the other configurations in the following list with
	# data = datasets.load_dataset('my_dataset', 'first_domain')
	# data = datasets.load_dataset('my_dataset', 'second_domain')
	a			: List[Any]      =				[
	    datasets.BuilderConfig(name="first_domain"       ,					version=lowerCAmelCase__       ,					description="This part of my dataset covers a first domain"  ),
	    datasets.BuilderConfig(name="second_domain"       ,					version=lowerCAmelCase__       ,					description="This part of my dataset covers a second domain"  ),
	]

	a			: Dict      =				"first_domain"  # It's not mandatory to have a default configuration. Just use one if it make sense.


	def   _UpperCAmelCase   (self       )							->							List[str]:







				'''simple docstring'''





				if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
							__lowercase   =							datasets.Features(
							    {
							        '''sentence''': datasets.Value('''string'''       ),
							        '''option1''': datasets.Value('''string'''       ),
							        '''answer''': datasets.Value('''string'''       )
							        # These are the features of your dataset like images, labels ...
							    }       )
				else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
							__lowercase   =							datasets.Features(
							    {
							        '''sentence''': datasets.Value('''string'''       ),
							        '''option2''': datasets.Value('''string'''       ),
							        '''second_domain_answer''': datasets.Value('''string'''       )
							        # These are the features of your dataset like images, labels ...
							    }       )
				return datasets.DatasetInfo(
				    # This is the description that will appear on the datasets page.
				    description=_DESCRIPTION   ,features=_lowerCamelCase   ,homepage=_HOMEPAGE   ,license=_LICENSE   ,citation=_CITATION   ,)


	def   _UpperCAmelCase   (self   ,_lowerCamelCase       )							->							Dict:







				'''simple docstring'''





				__lowercase   =							_URLS[self.config.name]
				__lowercase   =							dl_manager.download_and_extract(_lowerCamelCase       )
				return [
				    datasets.SplitGenerator(
				        name=datasets.Split.TRAIN   ,gen_kwargs={
				            '''filepath''': os.path.join(_lowerCamelCase   ,'''train.jsonl'''       ),
				            '''split''': '''train''',
				        }   ,),
				    datasets.SplitGenerator(
				        name=datasets.Split.VALIDATION   ,gen_kwargs={
				            '''filepath''': os.path.join(_lowerCamelCase   ,'''dev.jsonl'''       ),
				            '''split''': '''dev''',
				        }   ,),
				    datasets.SplitGenerator(
				        name=datasets.Split.TEST   ,gen_kwargs={
				            '''filepath''': os.path.join(_lowerCamelCase   ,'''test.jsonl'''       ),
				            '''split''': '''test'''
				        }   ,),
				]


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase       )							->							str:







				'''simple docstring'''





				with open(_lowerCamelCase   ,encoding='''utf-8'''       ) as f:
							for key, row in enumerate(_lowerCamelCase       ):
										__lowercase   =							json.loads(_lowerCamelCase       )
										if self.config.name == "first_domain":
													# Yields examples as (key, example) tuples
													yield key, {
													    "sentence": data["sentence"],
													    "option1": data["option1"],
													    "answer": "" if split == "test" else data["answer"],
													}
										else:
													yield key, {
													    "sentence": data["sentence"],
													    "option2": data["option2"],
													    "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
													}





