
import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
							import torch

							from ..models.auto.modeling_auto import (
							    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
							    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
							)

if is_tf_available() and is_tensorflow_probability_available():
							import tensorflow as tf
							import tensorflow_probability as tfp

							from ..models.auto.modeling_tf_auto import (
							    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
							    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
							)

class 				_UpperCamelCase (    _A     ):






			'''simple docstring'''

			def __call__(				self       :				Tuple  ,		snake_case_       :				Tuple=None  ,		snake_case_       :				int=None  ,		**snake_case_       :				Optional[Any]					):
							# Returns tqa_pipeline_inputs of shape:
							# [
							#   {"table": pd.DataFrame, "query": List[str]},
							#   ...,
							#   {"table": pd.DataFrame, "query" : List[str]}
							# ]
							requires_backends(self  ,		"""pandas"""					)
							import pandas as pd

							if table is None:
											raise ValueError("""Keyword argument `table` cannot be None."""					)
							elif query is None:
											if isinstance(snake_case_  ,		snake_case_					) and table.get("""query"""					) is not None and table.get("""table"""					) is not None:
															UpperCamelCase_:							Tuple		=  [table]
											elif isinstance(snake_case_  ,		snake_case_					) and len(snake_case_					) > 0:
															if not all(isinstance(snake_case_  ,		snake_case_					) for d in table					):
																			raise ValueError(
																			    f'''Keyword argument `table` should be a list of dict, but is {(type(snake_case_					) for d in table)}'''					)

															if table[0].get("""query"""					) is not None and table[0].get("""table"""					) is not None:
																			UpperCamelCase_:							str		=  table
															else:
																			raise ValueError(
																			    """If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"""
																			    f''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.'''					)
											elif Dataset is not None and isinstance(snake_case_  ,		snake_case_					) or isinstance(snake_case_  ,		types.GeneratorType					):
															return table
											else:
															raise ValueError(
															    """Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but """
															    f'''is {type(snake_case_					)})'''					)
							else:
											UpperCamelCase_:							Tuple		=  [{"""table""": table, """query""": query}]

							for tqa_pipeline_input in tqa_pipeline_inputs:
											if not isinstance(tqa_pipeline_input["""table"""]  ,		pd.DataFrame					):
															if tqa_pipeline_input["table"] is None:
																			raise ValueError("""Table cannot be None."""					)

															UpperCamelCase_:							Any		=  pd.DataFrame(tqa_pipeline_input["""table"""]					)

							return tqa_pipeline_inputs




@add_end_docstrings(_A     )
class 				_UpperCamelCase (    _A     ):






			'''simple docstring'''

			__UpperCamelCase					:  int    					=       """table,query"""

			def __init__(				self       :				Union[str, Any]  ,		snake_case_       :				str=TableQuestionAnsweringArgumentHandler()  ,		*snake_case_       :				Any  ,		**snake_case_       :				Tuple					):
							super().__init__(*snake_case_  ,		**snake_case_					)
							UpperCamelCase_:							Dict		=  args_parser

							self.check_model_type(
							    dict(
							        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
							        + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()					)
							    if self.framework == """tf"""
							    else dict(
							        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()					)					)

							UpperCamelCase_:							Union[str, Any]		=  bool(getattr(self.model.config  ,		"""aggregation_labels"""  ,		snake_case_					)					) and bool(
							    getattr(self.model.config  ,		"""num_aggregation_labels"""  ,		snake_case_					)					)
							UpperCamelCase_:							int		=  """tapas""" if hasattr(self.model.config  ,		"""aggregation_labels"""					) else None

			def 					lowerCAmelCase__		(				self       :				Union[str, Any]  ,		**snake_case_       :				List[str]					):
							return self.model(**snake_case_					)

			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		**snake_case_       :				Optional[Any]					):
							if self.framework == "pt":
											UpperCamelCase_:							Optional[Any]		=  []
											UpperCamelCase_:							Tuple		=  []
											UpperCamelCase_:							str		=  None
											UpperCamelCase_:							Any		=  inputs["""input_ids"""].shape[0]

											UpperCamelCase_:							Dict		=  inputs["""input_ids"""].to(self.device					)
											UpperCamelCase_:							int		=  inputs["""attention_mask"""].to(self.device					)
											UpperCamelCase_:							Dict		=  inputs["""token_type_ids"""].to(self.device					)
											UpperCamelCase_:							int		=  None

											for index in range(snake_case_					):
															# If sequences have already been processed, the token type IDs will be created according to the previous
															# answer.
															if prev_answers is not None:
																			UpperCamelCase_:							Tuple		=  token_type_ids_example[:, 3]  # shape (seq_len,)
																			UpperCamelCase_:							Any		=  np.zeros_like(prev_labels_example.cpu().numpy()					)  # shape (seq_len,)

																			UpperCamelCase_:							int		=  token_type_ids[index]  # shape (seq_len, 7)
																			for i in range(model_labels.shape[0]					):
																							UpperCamelCase_:							Union[str, Any]		=  token_type_ids_example[:, 0].tolist()[i]
																							UpperCamelCase_:							List[Any]		=  token_type_ids_example[:, 1].tolist()[i] - 1
																							UpperCamelCase_:							Optional[int]		=  token_type_ids_example[:, 2].tolist()[i] - 1

																							if row_id >= 0 and col_id >= 0 and segment_id == 1:
																											UpperCamelCase_:							Optional[Any]		=  int(prev_answers[(col_id, row_id)]					)

																			UpperCamelCase_:							int		=  torch.from_numpy(snake_case_					).type(torch.long					).to(self.device					)

															UpperCamelCase_:							Optional[Any]		=  input_ids[index]
															UpperCamelCase_:							Optional[int]		=  attention_mask[index]  # shape (seq_len,)
															UpperCamelCase_:							List[str]		=  token_type_ids[index]  # shape (seq_len, 7)
															UpperCamelCase_:							List[str]		=  self.model(
															    input_ids=input_ids_example.unsqueeze(0					)  ,		attention_mask=attention_mask_example.unsqueeze(0					)  ,		token_type_ids=token_type_ids_example.unsqueeze(0					)  ,		)
															UpperCamelCase_:							List[str]		=  outputs.logits

															if self.aggregate:
																			all_aggregations.append(outputs.logits_aggregation					)

															all_logits.append(snake_case_					)

															UpperCamelCase_:							Tuple		=  torch.distributions.Bernoulli(logits=snake_case_					)
															UpperCamelCase_:							int		=  dist_per_token.probs * attention_mask_example.type(torch.floataa					).to(
															    dist_per_token.probs.device					)

															UpperCamelCase_:							Tuple		=  collections.defaultdict(snake_case_					)
															for i, p in enumerate(probabilities.squeeze().tolist()					):
																			UpperCamelCase_:							int		=  token_type_ids_example[:, 0].tolist()[i]
																			UpperCamelCase_:							Any		=  token_type_ids_example[:, 1].tolist()[i] - 1
																			UpperCamelCase_:							Optional[Any]		=  token_type_ids_example[:, 2].tolist()[i] - 1
																			if col >= 0 and row >= 0 and segment_id == 1:
																							coords_to_probs[(col, row)].append(snake_case_					)

															UpperCamelCase_:							str		=  {key: np.array(coords_to_probs[key]					).mean() > 0.5 for key in coords_to_probs}

											UpperCamelCase_:							int		=  torch.cat(tuple(snake_case_					)  ,		0					)

											return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(snake_case_					)  ,		0					))
							else:
											UpperCamelCase_:							Any		=  []
											UpperCamelCase_:							Tuple		=  []
											UpperCamelCase_:							Optional[int]		=  None
											UpperCamelCase_:							Any		=  inputs["""input_ids"""].shape[0]

											UpperCamelCase_:							Union[str, Any]		=  inputs["""input_ids"""]
											UpperCamelCase_:							Optional[Any]		=  inputs["""attention_mask"""]
											UpperCamelCase_:							Optional[Any]		=  inputs["""token_type_ids"""].numpy()
											UpperCamelCase_:							Dict		=  None

											for index in range(snake_case_					):
															# If sequences have already been processed, the token type IDs will be created according to the previous
															# answer.
															if prev_answers is not None:
																			UpperCamelCase_:							Tuple		=  token_type_ids_example[:, 3]  # shape (seq_len,)
																			UpperCamelCase_:							str		=  np.zeros_like(snake_case_  ,		dtype=np.intaa					)  # shape (seq_len,)

																			UpperCamelCase_:							str		=  token_type_ids[index]  # shape (seq_len, 7)
																			for i in range(model_labels.shape[0]					):
																							UpperCamelCase_:							List[Any]		=  token_type_ids_example[:, 0].tolist()[i]
																							UpperCamelCase_:							List[str]		=  token_type_ids_example[:, 1].tolist()[i] - 1
																							UpperCamelCase_:							Optional[int]		=  token_type_ids_example[:, 2].tolist()[i] - 1

																							if row_id >= 0 and col_id >= 0 and segment_id == 1:
																											UpperCamelCase_:							List[str]		=  int(prev_answers[(col_id, row_id)]					)

																			UpperCamelCase_:							int		=  model_labels

															UpperCamelCase_:							int		=  input_ids[index]
															UpperCamelCase_:							List[Any]		=  attention_mask[index]  # shape (seq_len,)
															UpperCamelCase_:							Tuple		=  token_type_ids[index]  # shape (seq_len, 7)
															UpperCamelCase_:							int		=  self.model(
															    input_ids=np.expand_dims(snake_case_  ,		axis=0					)  ,		attention_mask=np.expand_dims(snake_case_  ,		axis=0					)  ,		token_type_ids=np.expand_dims(snake_case_  ,		axis=0					)  ,		)
															UpperCamelCase_:							Union[str, Any]		=  outputs.logits

															if self.aggregate:
																			all_aggregations.append(outputs.logits_aggregation					)

															all_logits.append(snake_case_					)

															UpperCamelCase_:							List[str]		=  tfp.distributions.Bernoulli(logits=snake_case_					)
															UpperCamelCase_:							Tuple		=  dist_per_token.probs_parameter() * tf.cast(snake_case_  ,		tf.floataa					)

															UpperCamelCase_:							Optional[Any]		=  collections.defaultdict(snake_case_					)
															UpperCamelCase_:							Any		=  token_type_ids_example
															for i, p in enumerate(tf.squeeze(snake_case_					).numpy().tolist()					):
																			UpperCamelCase_:							Optional[int]		=  token_type_ids_example[:, 0].tolist()[i]
																			UpperCamelCase_:							Any		=  token_type_ids_example[:, 1].tolist()[i] - 1
																			UpperCamelCase_:							int		=  token_type_ids_example[:, 2].tolist()[i] - 1
																			if col >= 0 and row >= 0 and segment_id == 1:
																							coords_to_probs[(col, row)].append(snake_case_					)

															UpperCamelCase_:							str		=  {key: np.array(coords_to_probs[key]					).mean() > 0.5 for key in coords_to_probs}

											UpperCamelCase_:							Dict		=  tf.concat(tuple(snake_case_					)  ,		0					)

											return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(snake_case_					)  ,		0					))

			def __call__(				self       :				List[str]  ,		*snake_case_       :				Optional[Any]  ,		**snake_case_       :				Dict					):
							UpperCamelCase_:							List[str]		=  self._args_parser(*snake_case_  ,		**snake_case_					)

							UpperCamelCase_:							List[Any]		=  super().__call__(snake_case_  ,		**snake_case_					)
							if len(snake_case_					) == 1:
											return results[0]
							return results

			def 					lowerCAmelCase__		(				self       :				Any  ,		snake_case_       :				List[str]=None  ,		snake_case_       :				Tuple=None  ,		snake_case_       :				List[Any]=None  ,		**snake_case_       :				str					):
							UpperCamelCase_:							str		=  {}
							if padding is not None:
											UpperCamelCase_:							Union[str, Any]		=  padding
							if truncation is not None:
											UpperCamelCase_:							int		=  truncation

							UpperCamelCase_:							Tuple		=  {}
							if sequential is not None:
											UpperCamelCase_:							Any		=  sequential
							return preprocess_params, forward_params, {}

			def 					lowerCAmelCase__		(				self       :				str  ,		snake_case_       :				List[str]  ,		snake_case_       :				List[str]=None  ,		snake_case_       :				List[str]=True  ,		snake_case_       :				Union[str, Any]=None					):
							if truncation is None:
											if self.type == "tapas":
															UpperCamelCase_:							Optional[int]		=  """drop_rows_to_fit"""
											else:
															UpperCamelCase_:							Dict		=  """do_not_truncate"""

							UpperCamelCase_,      UpperCamelCase_:							Tuple		=  pipeline_input["""table"""], pipeline_input["""query"""]
							if table.empty:
											raise ValueError("""table is empty"""					)
							if query is None or query == "":
											raise ValueError("""query is empty"""					)
							UpperCamelCase_:							List[Any]		=  self.tokenizer(snake_case_  ,		snake_case_  ,		return_tensors=self.framework  ,		truncation=snake_case_  ,		padding=snake_case_					)
							UpperCamelCase_:							Tuple		=  table
							return inputs

			def 					lowerCAmelCase__		(				self       :				int  ,		snake_case_       :				int  ,		snake_case_       :				Tuple=False					):
							UpperCamelCase_:							Optional[Any]		=  model_inputs.pop("""table"""					)

							if self.type == "tapas":
											if sequential:
															UpperCamelCase_:							Dict		=  self.sequential_inference(**snake_case_					)
											else:
															UpperCamelCase_:							Any		=  self.batch_inference(**snake_case_					)
							else:
											UpperCamelCase_:							Optional[Any]		=  self.model.generate(**snake_case_					)
							UpperCamelCase_:							Union[str, Any]		=  {"""model_inputs""": model_inputs, """table""": table, """outputs""": outputs}
							return model_outputs




			def 					lowerCAmelCase__		(				self       :				List[str]  ,		snake_case_       :				List[Any]					):
							UpperCamelCase_:							int		=  model_outputs["""model_inputs"""]
							UpperCamelCase_:							List[str]		=  model_outputs["""table"""]
							UpperCamelCase_:							Any		=  model_outputs["""outputs"""]
							if self.type == "tapas":
											if self.aggregate:
															UpperCamelCase_,      UpperCamelCase_:							Union[str, Any]		=  outputs[:2]
															UpperCamelCase_:							List[Any]		=  self.tokenizer.convert_logits_to_predictions(snake_case_  ,		snake_case_  ,		snake_case_					)
															UpperCamelCase_,      UpperCamelCase_:							List[Any]		=  predictions
															UpperCamelCase_:							List[Any]		=  {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(snake_case_					)}

															UpperCamelCase_:							Any		=  self.model.config.no_aggregation_label_index
															UpperCamelCase_:							Any		=  {
															    i: aggregators[i] + """ > """ for i, pred in enumerate(snake_case_					) if pred != no_agg_label_index
															}
											else:
															UpperCamelCase_:							Optional[int]		=  outputs[0]
															UpperCamelCase_:							Dict		=  self.tokenizer.convert_logits_to_predictions(snake_case_  ,		snake_case_					)
															UpperCamelCase_:							List[str]		=  predictions[0]
															UpperCamelCase_:							Optional[Any]		=  {}
															UpperCamelCase_:							Union[str, Any]		=  {}
											UpperCamelCase_:							Optional[Any]		=  []
											for index, coordinates in enumerate(snake_case_					):
															UpperCamelCase_:							Any		=  [table.iat[coordinate] for coordinate in coordinates]
															UpperCamelCase_:							List[Any]		=  aggregators.get(snake_case_  ,		""""""					)
															UpperCamelCase_:							Dict		=  aggregators_prefix.get(snake_case_  ,		""""""					)
															UpperCamelCase_:							Optional[Any]		=  {
															    """answer""": aggregator_prefix + """, """.join(snake_case_					),
															    """coordinates""": coordinates,
															    """cells""": [table.iat[coordinate] for coordinate in coordinates],
															}
															if aggregator:
																			UpperCamelCase_:							Optional[Any]		=  aggregator

															answers.append(snake_case_					)
											if len(snake_case_					) == 0:
															raise PipelineException("""Empty answer"""					)
							else:
											UpperCamelCase_:							List[str]		=  [{"""answer""": answer} for answer in self.tokenizer.batch_decode(snake_case_  ,		skip_special_tokens=snake_case_					)]

							return answers if len(snake_case_					) > 1 else answers[0]








import torch

from transformers import AutoTokenizer

class 				_UpperCamelCase (    _A     ):






			'''simple docstring'''

			def __init__(				self       :				List[str]  ,		snake_case_       :				Dict					):
							UpperCamelCase_:							int		=  AutoTokenizer.from_pretrained(snake_case_					)

			def 					lowerCAmelCase__		(				self       :				Union[str, Any]  ,		snake_case_       :				Optional[Any]					):

							if isinstance(snake_case_  ,		snake_case_					) and all(isinstance(_x  ,		snake_case_					) for _x in x					):
											UpperCamelCase_:							Tuple		=  None
											for l in x:
															UpperCamelCase_:							Optional[Any]		=  self.tokenizer(
															    snake_case_  ,		padding="""max_length"""  ,		max_length=384  ,		truncation=snake_case_  ,		return_tensors="""pt"""  ,		)
															UpperCamelCase_:							str		=  torch.tensor([len(snake_case_					)]					)
															if d is not None:
																			for k in d.keys():
																							UpperCamelCase_:							List[Any]		=  torch.cat((d[k], t[k])  ,		0					)
															else:
																			UpperCamelCase_:							Optional[int]		=  t

											UpperCamelCase_:							Union[str, Any]		=  torch.tensor(self.tokenizer.convert_tokens_to_ids("""[E]"""					)					)
											UpperCamelCase_:							List[str]		=  torch.tensor(self.tokenizer.convert_tokens_to_ids("""[/E]"""					)					)

							elif isinstance(snake_case_  ,		snake_case_					) and all(isinstance(_x  ,		snake_case_					) for _x in x					):
											UpperCamelCase_:							List[str]		=  self.tokenizer(
											    snake_case_  ,		padding="""max_length"""  ,		max_length=384  ,		truncation=snake_case_  ,		return_tensors="""pt"""  ,		)

							else:
											raise Exception(
											    """Type of parameter x was not recognized! Only `list of strings` for query or `list of lists of"""
											    """ strings` for supports are supported."""					)

							return d




			def 					lowerCAmelCase__		(				self       :				int  ,		snake_case_       :				List[str]  ,		snake_case_       :				Optional[int]  ,		snake_case_       :				Optional[Any]  ,		snake_case_       :				Optional[int]  ,		snake_case_       :				List[Any]=0.70					):
							UpperCamelCase_:							int		=  []
							for idx in range(len(W_query["""input_ids"""]					)					):
											UpperCamelCase_:							List[str]		=  range(p_start.shape[1]					)

											UpperCamelCase_:							str		=  []
											for start_id in start_indexes:
															for end_id in end_indexes:
																			if start_id < end_id:
																							output.append(
																							    (
																							        start_id,
																							        end_id,
																							        p_start[idx][start_id].item(),
																							        p_end[idx][end_id].item(),
																							    )					)

											output.sort(key=lambda snake_case_					: (tup[2] * tup[3])  ,		reverse=snake_case_					)
											UpperCamelCase_:							Optional[int]		=  []
											for k in range(len(snake_case_					)					):
															if output[k][2] * output[k][3] >= thresh:
																			UpperCamelCase_,      UpperCamelCase_:							Tuple		=  output[k][0], output[k][1]
																			UpperCamelCase_:							Any		=  self.tokenizer.decode(W_query["""input_ids"""][idx][c_start_pos:c_end_pos]					)
																			temp.append((decoded, output[k][2] * output[k][3])					)

											final_outputs.append(snake_case_					)

							return final_outputs






