

"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__A     :    str    =		logging.get_logger(__name__)

__A     :    int    =		{'vocab_file': 'vocab.txt'}

__A     :    Dict    =		{
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

__A     :    Tuple    =		{
    'tau/splinter-base': 5_12,
    'tau/splinter-base-qass': 5_12,
    'tau/splinter-large': 5_12,
    'tau/splinter-large-qass': 5_12,
}

__A     :    Union[str, Any]    =		{
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}





class 			__UpperCamelCase (   _A						):
		SCREAMING_SNAKE_CASE								=    VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE								=    PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE								=    PRETRAINED_INIT_CONFIGURATION
		SCREAMING_SNAKE_CASE								=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE								=    SplinterTokenizer


		def __init__(self :		Optional[Any]     ,     __SCREAMING_SNAKE_CASE :		Optional[Any]=None     ,     __SCREAMING_SNAKE_CASE :		int=None     ,     __SCREAMING_SNAKE_CASE :		List[Any]=True     ,     __SCREAMING_SNAKE_CASE :		Tuple="[UNK]"     ,     __SCREAMING_SNAKE_CASE :		List[Any]="[SEP]"     ,     __SCREAMING_SNAKE_CASE :		Any="[PAD]"     ,     __SCREAMING_SNAKE_CASE :		Dict="[CLS]"     ,     __SCREAMING_SNAKE_CASE :		Tuple="[MASK]"     ,     __SCREAMING_SNAKE_CASE :		Tuple="[QUESTION]"     ,     __SCREAMING_SNAKE_CASE :		List[Any]=True     ,     __SCREAMING_SNAKE_CASE :		Optional[Any]=None     ,     **__SCREAMING_SNAKE_CASE :		Union[str, Any]     ,     ):
							super().__init__(
							    __SCREAMING_SNAKE_CASE     ,     tokenizer_file=__SCREAMING_SNAKE_CASE     ,     do_lower_case=__SCREAMING_SNAKE_CASE     ,     unk_token=__SCREAMING_SNAKE_CASE     ,     sep_token=__SCREAMING_SNAKE_CASE     ,     pad_token=__SCREAMING_SNAKE_CASE     ,     cls_token=__SCREAMING_SNAKE_CASE     ,     mask_token=__SCREAMING_SNAKE_CASE     ,     tokenize_chinese_chars=__SCREAMING_SNAKE_CASE     ,     strip_accents=__SCREAMING_SNAKE_CASE     ,     additional_special_tokens=(question_token,)     ,     **__SCREAMING_SNAKE_CASE     ,     )

							A			=					json.loads(self.backend_tokenizer.normalizer.__getstate__())
							if (
							    pre_tok_state.get("lowercase"     ,     __SCREAMING_SNAKE_CASE) != do_lower_case
							    or pre_tok_state.get("strip_accents"     ,     __SCREAMING_SNAKE_CASE) != strip_accents
							):
												A			=					getattr(__SCREAMING_SNAKE_CASE     ,     pre_tok_state.pop("type"))
												A			=					do_lower_case
												A			=					strip_accents
												A			=					pre_tok_class(**__SCREAMING_SNAKE_CASE)

							A			=					do_lower_case


		@property
		def 					SCREAMING_SNAKE_CASE__   (self :		Optional[int]):
							return self.convert_tokens_to_ids(self.question_token)


		def 					SCREAMING_SNAKE_CASE__   (self :		Optional[int]     ,     __SCREAMING_SNAKE_CASE :		List[int]     ,     __SCREAMING_SNAKE_CASE :		Optional[List[int]] = None):
							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							A			=					[self.cls_token_id]
							A			=					[self.sep_token_id]
							A			=					[self.question_token_id] + [self.convert_tokens_to_ids(".")]
							if self.padding_side == "right":
												# Input is question-then-context
												return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
							else:
												# Input is context-then-question
												return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


		def 					SCREAMING_SNAKE_CASE__   (self :		Tuple     ,     __SCREAMING_SNAKE_CASE :		List[int]     ,     __SCREAMING_SNAKE_CASE :		Optional[List[int]] = None):
							A			=					[self.sep_token_id]
							A			=					[self.cls_token_id]
							A			=					[self.question_token_id] + [self.convert_tokens_to_ids(".")]
							if token_ids_a is None:
												return len(cls + token_ids_a + sep) * [0]

							if self.padding_side == "right":
												# Input is question-then-context
												return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
							else:
												# Input is context-then-question
												return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]


		def 					SCREAMING_SNAKE_CASE__   (self :		Tuple     ,     __SCREAMING_SNAKE_CASE :		str     ,     __SCREAMING_SNAKE_CASE :		Optional[str] = None):
							A			=					self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,     name=__SCREAMING_SNAKE_CASE)
							return tuple(__SCREAMING_SNAKE_CASE)








"""simple docstring"""




import dataclasses
import enum
import functools
import math
import re

# The following script is adapted from the script of TaPas.
# Original: https://github.com/google-research/tapas/master/wikisql_utils.py
from typing import Any, List, Text


__A     :    int    =		'none'
__A     :    List[Any]    =		'none'
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__      ):
					"""simple docstring"""
					A			=					value.split(lowercase__      )
					return len(lowercase__      ) > 1 and any((len(lowercase__      ) == 3 for x in split)      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__      ):
					"""simple docstring"""
					if isinstance(lowercase__			,							lowercase__      ):
										return value
					if isinstance(lowercase__			,							lowercase__      ):
										return float(lowercase__      )
					if not isinstance(lowercase__			,							lowercase__      ):
										raise ValueError("Argument value is not a string. Can't parse it as float"      )
					A			=					value

					try:
										# Example: 1,000.7
										if "." in sanitized and "," in sanitized:
															return float(sanitized.replace(","			,							""      )      )
										# 1,000
										if "," in sanitized and _split_thousands(","			,							lowercase__      ):
															return float(sanitized.replace(","			,							""      )      )
										# 5,5556
										if "," in sanitized and sanitized.count(","      ) == 1 and not _split_thousands(","			,							lowercase__      ):
															return float(sanitized.replace(","			,							"."      )      )
										# 0.0.0.1
										if sanitized.count("."      ) > 1:
															return float(sanitized.replace("."			,							""      )      )
										# 0,0,0,1
										if sanitized.count(","      ) > 1:
															return float(sanitized.replace(","			,							""      )      )
										return float(lowercase__      )
					except ValueError:
										# Avoid adding the sanitized value in the error message.
										raise ValueError("Unable to convert value to float"      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__      ):
					"""simple docstring"""
					if answer is None:
										return None
					try:
										A			=					convert_to_float(lowercase__      )
										if isinstance(lowercase__			,							lowercase__      ) and math.isnan(lowercase__      ):
															return None
										return value
					except ValueError:
										return answer.lower()


__A     :    List[str]    =		{
    'text': lambda x: x,
    'real': convert_to_float,
}





class 			__UpperCamelCase (   enum.Enum						):
		SCREAMING_SNAKE_CASE								=    0
		SCREAMING_SNAKE_CASE								=    1
		SCREAMING_SNAKE_CASE								=    2
		SCREAMING_SNAKE_CASE								=    3
		SCREAMING_SNAKE_CASE								=    4
		SCREAMING_SNAKE_CASE								=    5





class 			__UpperCamelCase (   enum.Enum						):
		SCREAMING_SNAKE_CASE								=    0
		SCREAMING_SNAKE_CASE								=    1
		SCREAMING_SNAKE_CASE								=    2





@dataclasses.dataclass
class 			__UpperCamelCase :
		SCREAMING_SNAKE_CASE								=    42
		SCREAMING_SNAKE_CASE								=    42
		SCREAMING_SNAKE_CASE								=    42

__A     :    int    =		re.compile(R'\w+|[^\w\s]+', re.UNICODE | re.MULTILINE | re.DOTALL)
def    __SCREAMING_SNAKE_CASE						(						lowercase__      ):
					"""simple docstring"""
					return list(_TOKENIZER.findall(x.lower()      )      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__			,							lowercase__      ):
					"""simple docstring"""
					if operator == _Operator.EQUALS:
										return src == tgt
					elif operator == _Operator.GREATER:
										return src > tgt
					elif operator == _Operator.LESSER:
										return src < tgt
					raise ValueError(F"""Unknown operator: {operator}"""      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__			,							lowercase__      ):
					"""simple docstring"""
					A			=					table["types"]
					return _TYPE_CONVERTER[types[column]](lowercase__      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__      ):
					"""simple docstring"""
					return isinstance(lowercase__			,							lowercase__      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__			,							lowercase__      ):
					"""simple docstring"""
					for cond in conditions:
										A			=					row[cond.column]

										A			=					_parse_value(lowercase__			,							cond.column			,							cond.cmp_value      )

										if _is_string(lowercase__      ) and _is_string(lowercase__      ):
															A			=					_normalize_for_match(lowercase__      )
															A			=					_normalize_for_match(lowercase__      )

										if not isinstance(lowercase__			,							type(lowercase__      )      ):
															raise ValueError("Type difference {} != {}".format(type(lowercase__      )			,							type(lowercase__      )      )      )

										if not _compare(cond.operator			,							lowercase__			,							lowercase__      ):
															return False
					return True
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__			,							lowercase__      ):
					"""simple docstring"""
					if not answer_coordinates:
										if aggregation_op == _Aggregation.COUNT:
															return 0.0
										else:
															return EMPTY_ANSWER_AGG

    # Count can support non numeric answers.
					if aggregation_op == _Aggregation.COUNT:
										return float(len(lowercase__      )      )

					# If we have just one answer, if float returns it or try a conversion.
					A			=					[table["rows"][i][j] for (i, j) in answer_coordinates]
					if len(lowercase__      ) == 1:
										try:
															return convert_to_float(values[0]      )
										except ValueError as e:
															if aggregation_op != _Aggregation.NONE:
																				raise e

					if aggregation_op == _Aggregation.NONE:
										return None

					# Other aggregation only support numeric values. Bail out if we have strings.
					if not all((isinstance(lowercase__			,							(int, float)      ) for v in values)      ):
										return None

					if aggregation_op == _Aggregation.SUM:
										return float(sum(lowercase__      )      )
					elif aggregation_op == _Aggregation.AVERAGE:
										return sum(lowercase__      ) / len(lowercase__      )
					else:
										raise ValueError(F"""Unknown aggregation: {aggregation_op}"""      )
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__      ):
					"""simple docstring"""
					A			=					sql_query["agg"]
					if aggregation_op_index >= 3:
										A			=					_Aggregation(lowercase__      )
					else:
										A			=					_Aggregation.NONE

					A			=					sql_query["sel"]
					A			=					[
					    _Condition(lowercase__			,							_Operator(lowercase__      )			,							lowercase__      )
					    for column, operator, cmp_value in zip(
					        sql_query["conds"]["column_index"]			,							sql_query["conds"]["operator_index"]			,							sql_query["conds"]["condition"]      )
					]

					A			=					[]
					for row in range(len(table["rows"]      )      ):
										if _respect_conditions(lowercase__			,							table["rows"][row]			,							lowercase__      ):
															indices.append((row, target_column)      )

					if not indices:
										return [], aggregation_op

					if len(lowercase__      ) == 1:
										return indices, aggregation_op

					# Parsing of MIN/MAX.
					if aggregation_op_index in (1, 2):
										A			=					{2: min, 1: max}
										A			=					[(table["rows"][i][j], index) for index, (i, j) in enumerate(lowercase__      )]
										A			=					functools.reduce(operators[sql_query["agg"]]			,							lowercase__      )

										A			=					[indices[reduced[1]]]
										return ret, _Aggregation.NONE

					return indices, aggregation_op
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__			,							lowercase__      ):
					"""simple docstring"""
					if float_answer is not None:
										return [str(lowercase__      )]
					return [str(table["real_rows"][r][c]      ) for r, c in answer_coordinates]
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							lowercase__      ):
					"""simple docstring"""
					A			,    A			=					_get_answer_coordinates(lowercase__			,							lowercase__      )
					A			=					_get_float_answer(lowercase__			,							lowercase__			,							lowercase__      )
					A			=					_get_answer_text(lowercase__			,							lowercase__			,							lowercase__      )
					# keep the original data the same with TaPas
					if len(lowercase__      ) == 0:
										A			=					[EMPTY_ANSWER]
					return answer_text





