import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
	import torch

	from ..models.auto.modeling_auto import (
	    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
	    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
	)

if is_tf_available() and is_tensorflow_probability_available():
	import tensorflow as tf
	import tensorflow_probability as tfp

	from ..models.auto.modeling_tf_auto import (
	    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
	    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
	)


class 						a__							(   __snake_case		):



			def __call__(						self      ,						UpperCAmelCase=None      ,						UpperCAmelCase=None      ,						**UpperCAmelCase    ) ->    List[Any]:
								# Returns tqa_pipeline_inputs of shape:
								# [
								#   {"table": pd.DataFrame, "query": List[str]},
								#   ...,
								#   {"table": pd.DataFrame, "query" : List[str]}
								# ]
								requires_backends(self      ,						'pandas'    )
								import pandas as pd

								if table is None:
													raise ValueError('Keyword argument `table` cannot be None.'    )
								elif query is None:
													if isinstance(UpperCAmelCase      ,						UpperCAmelCase    ) and table.get('query'    ) is not None and table.get('table'    ) is not None:
																		__a							=   [table]
													elif isinstance(UpperCAmelCase      ,						UpperCAmelCase    ) and len(UpperCAmelCase    ) > 0:
																		if not all(isinstance(UpperCAmelCase      ,						UpperCAmelCase    ) for d in table    ):
																							raise ValueError(
																							    f'''Keyword argument `table` should be a list of dict, but is {(type(UpperCAmelCase    ) for d in table)}'''    )

																		if table[0].get('query'    ) is not None and table[0].get('table'    ) is not None:
																							__a							=   table
																		else:
																							raise ValueError(
																							    'If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`'
																							    f''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.'''    )
													elif Dataset is not None and isinstance(UpperCAmelCase      ,						UpperCAmelCase    ) or isinstance(UpperCAmelCase      ,						types.GeneratorType    ):
																		return table
													else:
																		raise ValueError(
																		    'Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but '
																		    f'''is {type(UpperCAmelCase    )})'''    )
								else:
													__a							=   [{'table': table, 'query': query}]

								for tqa_pipeline_input in tqa_pipeline_inputs:
													if not isinstance(tqa_pipeline_input['table']      ,						pd.DataFrame    ):
																		if tqa_pipeline_input["table"] is None:
																							raise ValueError('Table cannot be None.'    )

																		__a							=   pd.DataFrame(tqa_pipeline_input['table']    )

								return tqa_pipeline_inputs






@add_end_docstrings(__snake_case		)
class 						a__							(   __snake_case		):
			A__				:   Dict								=			'table,query'



			def __init__(						self      ,						UpperCAmelCase=TableQuestionAnsweringArgumentHandler()      ,						*UpperCAmelCase      ,						**UpperCAmelCase    ) ->    Optional[int]:
								super().__init__(*UpperCAmelCase      ,						**UpperCAmelCase    )
								__a							=   args_parser

								self.check_model_type(
								    dict(
								        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
								        + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()    )
								    if self.framework == 'tf'
								    else dict(
								        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()    )    )

								__a							=   bool(getattr(self.model.config      ,						'aggregation_labels'      ,						UpperCAmelCase    )    ) and bool(
								    getattr(self.model.config      ,						'num_aggregation_labels'      ,						UpperCAmelCase    )    )
								__a							=   'tapas' if hasattr(self.model.config      ,						'aggregation_labels'    ) else None



			def __SCREAMING_SNAKE_CASE							(						self      ,						**UpperCAmelCase    ) ->    List[Any]:
								return self.model(**UpperCAmelCase    )



			def __SCREAMING_SNAKE_CASE							(						self      ,						**UpperCAmelCase    ) ->    Tuple:
								if self.framework == "pt":
													__a							=   []
													__a							=   []
													__a							=   None
													__a							=   inputs['input_ids'].shape[0]

													__a							=   inputs['input_ids'].to(self.device    )
													__a							=   inputs['attention_mask'].to(self.device    )
													__a							=   inputs['token_type_ids'].to(self.device    )
													__a							=   None

													for index in range(UpperCAmelCase    ):
																		# If sequences have already been processed, the token type IDs will be created according to the previous
																		# answer.
																		if prev_answers is not None:
																							__a							=   token_type_ids_example[:, 3]  # shape (seq_len,)
																							__a							=   np.zeros_like(prev_labels_example.cpu().numpy()    )  # shape (seq_len,)

																							__a							=   token_type_ids[index]  # shape (seq_len, 7)
																							for i in range(model_labels.shape[0]    ):
																												__a							=   token_type_ids_example[:, 0].tolist()[i]
																												__a							=   token_type_ids_example[:, 1].tolist()[i] - 1
																												__a							=   token_type_ids_example[:, 2].tolist()[i] - 1

																												if row_id >= 0 and col_id >= 0 and segment_id == 1:
																																	__a							=   int(prev_answers[(col_id, row_id)]    )

																							__a							=   torch.from_numpy(UpperCAmelCase    ).type(torch.long    ).to(self.device    )

																		__a							=   input_ids[index]
																		__a							=   attention_mask[index]  # shape (seq_len,)
																		__a							=   token_type_ids[index]  # shape (seq_len, 7)
																		__a							=   self.model(
																		    input_ids=input_ids_example.unsqueeze(0    )      ,						attention_mask=attention_mask_example.unsqueeze(0    )      ,						token_type_ids=token_type_ids_example.unsqueeze(0    )      ,						)
																		__a							=   outputs.logits

																		if self.aggregate:
																							all_aggregations.append(outputs.logits_aggregation    )

																		all_logits.append(UpperCAmelCase    )

																		__a							=   torch.distributions.Bernoulli(logits=UpperCAmelCase    )
																		__a							=   dist_per_token.probs * attention_mask_example.type(torch.floataa    ).to(
																		    dist_per_token.probs.device    )

																		__a							=   collections.defaultdict(UpperCAmelCase    )
																		for i, p in enumerate(probabilities.squeeze().tolist()    ):
																							__a							=   token_type_ids_example[:, 0].tolist()[i]
																							__a							=   token_type_ids_example[:, 1].tolist()[i] - 1
																							__a							=   token_type_ids_example[:, 2].tolist()[i] - 1
																							if col >= 0 and row >= 0 and segment_id == 1:
																												coords_to_probs[(col, row)].append(UpperCAmelCase    )

																		__a							=   {key: np.array(coords_to_probs[key]    ).mean() > 0.5 for key in coords_to_probs}

													__a							=   torch.cat(tuple(UpperCAmelCase    )      ,						0    )

													return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(UpperCAmelCase    )      ,						0    ))
								else:
													__a							=   []
													__a							=   []
													__a							=   None
													__a							=   inputs['input_ids'].shape[0]

													__a							=   inputs['input_ids']
													__a							=   inputs['attention_mask']
													__a							=   inputs['token_type_ids'].numpy()
													__a							=   None

													for index in range(UpperCAmelCase    ):
																		# If sequences have already been processed, the token type IDs will be created according to the previous
																		# answer.
																		if prev_answers is not None:
																							__a							=   token_type_ids_example[:, 3]  # shape (seq_len,)
																							__a							=   np.zeros_like(UpperCAmelCase      ,						dtype=np.intaa    )  # shape (seq_len,)

																							__a							=   token_type_ids[index]  # shape (seq_len, 7)
																							for i in range(model_labels.shape[0]    ):
																												__a							=   token_type_ids_example[:, 0].tolist()[i]
																												__a							=   token_type_ids_example[:, 1].tolist()[i] - 1
																												__a							=   token_type_ids_example[:, 2].tolist()[i] - 1

																												if row_id >= 0 and col_id >= 0 and segment_id == 1:
																																	__a							=   int(prev_answers[(col_id, row_id)]    )

																							__a							=   model_labels

																		__a							=   input_ids[index]
																		__a							=   attention_mask[index]  # shape (seq_len,)
																		__a							=   token_type_ids[index]  # shape (seq_len, 7)
																		__a							=   self.model(
																		    input_ids=np.expand_dims(UpperCAmelCase      ,						axis=0    )      ,						attention_mask=np.expand_dims(UpperCAmelCase      ,						axis=0    )      ,						token_type_ids=np.expand_dims(UpperCAmelCase      ,						axis=0    )      ,						)
																		__a							=   outputs.logits

																		if self.aggregate:
																							all_aggregations.append(outputs.logits_aggregation    )

																		all_logits.append(UpperCAmelCase    )

																		__a							=   tfp.distributions.Bernoulli(logits=UpperCAmelCase    )
																		__a							=   dist_per_token.probs_parameter() * tf.cast(UpperCAmelCase      ,						tf.floataa    )

																		__a							=   collections.defaultdict(UpperCAmelCase    )
																		__a							=   token_type_ids_example
																		for i, p in enumerate(tf.squeeze(UpperCAmelCase    ).numpy().tolist()    ):
																							__a							=   token_type_ids_example[:, 0].tolist()[i]
																							__a							=   token_type_ids_example[:, 1].tolist()[i] - 1
																							__a							=   token_type_ids_example[:, 2].tolist()[i] - 1
																							if col >= 0 and row >= 0 and segment_id == 1:
																												coords_to_probs[(col, row)].append(UpperCAmelCase    )

																		__a							=   {key: np.array(coords_to_probs[key]    ).mean() > 0.5 for key in coords_to_probs}

													__a							=   tf.concat(tuple(UpperCAmelCase    )      ,						0    )

													return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(UpperCAmelCase    )      ,						0    ))



			def __call__(						self      ,						*UpperCAmelCase      ,						**UpperCAmelCase    ) ->    List[str]:
								__a							=   self._args_parser(*UpperCAmelCase      ,						**UpperCAmelCase    )

								__a							=   super().__call__(UpperCAmelCase      ,						**UpperCAmelCase    )
								if len(UpperCAmelCase    ) == 1:
													return results[0]
								return results



			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase=None      ,						UpperCAmelCase=None      ,						UpperCAmelCase=None      ,						**UpperCAmelCase    ) ->    str:
								__a							=   {}
								if padding is not None:
													__a							=   padding
								if truncation is not None:
													__a							=   truncation

								__a							=   {}
								if sequential is not None:
													__a							=   sequential
								return preprocess_params, forward_params, {}



			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase      ,						UpperCAmelCase=None      ,						UpperCAmelCase=True      ,						UpperCAmelCase=None    ) ->    Dict:
								if truncation is None:
													if self.type == "tapas":
																		__a							=   'drop_rows_to_fit'
													else:
																		__a							=   'do_not_truncate'

								__a					,      __a							=   pipeline_input['table'], pipeline_input['query']
								if table.empty:
													raise ValueError('table is empty'    )
								if query is None or query == "":
													raise ValueError('query is empty'    )
								__a							=   self.tokenizer(UpperCAmelCase      ,						UpperCAmelCase      ,						return_tensors=self.framework      ,						truncation=UpperCAmelCase      ,						padding=UpperCAmelCase    )
								__a							=   table
								return inputs



			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase      ,						UpperCAmelCase=False    ) ->    int:
								__a							=   model_inputs.pop('table'    )

								if self.type == "tapas":
													if sequential:
																		__a							=   self.sequential_inference(**UpperCAmelCase    )
													else:
																		__a							=   self.batch_inference(**UpperCAmelCase    )
								else:
													__a							=   self.model.generate(**UpperCAmelCase    )
								__a							=   {'model_inputs': model_inputs, 'table': table, 'outputs': outputs}
								return model_outputs







			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase    ) ->    List[str]:
								__a							=   model_outputs['model_inputs']
								__a							=   model_outputs['table']
								__a							=   model_outputs['outputs']
								if self.type == "tapas":
													if self.aggregate:
																		__a					,      __a							=   outputs[:2]
																		__a							=   self.tokenizer.convert_logits_to_predictions(UpperCAmelCase      ,						UpperCAmelCase      ,						UpperCAmelCase    )
																		__a					,      __a							=   predictions
																		__a							=   {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(UpperCAmelCase    )}

																		__a							=   self.model.config.no_aggregation_label_index
																		__a							=   {
																		    i: aggregators[i] + ' > ' for i, pred in enumerate(UpperCAmelCase    ) if pred != no_agg_label_index
																		}
													else:
																		__a							=   outputs[0]
																		__a							=   self.tokenizer.convert_logits_to_predictions(UpperCAmelCase      ,						UpperCAmelCase    )
																		__a							=   predictions[0]
																		__a							=   {}
																		__a							=   {}
													__a							=   []
													for index, coordinates in enumerate(UpperCAmelCase    ):
																		__a							=   [table.iat[coordinate] for coordinate in coordinates]
																		__a							=   aggregators.get(UpperCAmelCase      ,						''    )
																		__a							=   aggregators_prefix.get(UpperCAmelCase      ,						''    )
																		__a							=   {
																		    'answer': aggregator_prefix + ', '.join(UpperCAmelCase    ),
																		    'coordinates': coordinates,
																		    'cells': [table.iat[coordinate] for coordinate in coordinates],
																		}
																		if aggregator:
																							__a							=   aggregator

																		answers.append(UpperCAmelCase    )
													if len(UpperCAmelCase    ) == 0:
																		raise PipelineException('Empty answer'    )
								else:
													__a							=   [{'answer': answer} for answer in self.tokenizer.batch_decode(UpperCAmelCase      ,						skip_special_tokens=UpperCAmelCase    )]

								return answers if len(UpperCAmelCase    ) > 1 else answers[0]





from dataclasses import dataclass
from typing import Callable, Optional

import datasets


@dataclass
class 						a__							(   datasets.BuilderConfig		):
			A__				:   Optional[Callable]								=			None
			A__				:   Optional[dict]								=			None
			A__				:   Optional[datasets.Features]								=			None



			def __SCREAMING_SNAKE_CASE							(						self    ) ->    Any:
								assert self.generator is not None, "generator must be specified"

								if self.gen_kwargs is None:
													__a							=   {}






class 						a__							(   datasets.GeneratorBasedBuilder		):
			A__				:   List[Any]								=			GeneratorConfig



			def __SCREAMING_SNAKE_CASE							(						self    ) ->    Tuple:
								return datasets.DatasetInfo(features=self.config.features    )



			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase    ) ->    Optional[int]:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,						gen_kwargs=self.config.gen_kwargs    )]







			def __SCREAMING_SNAKE_CASE							(						self      ,						**UpperCAmelCase    ) ->    Tuple:
								for idx, ex in enumerate(self.config.generator(**UpperCAmelCase    )    ):
													yield idx, ex




