


'''simple docstring'''






import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__UpperCAmelCase    				=							"""block_records.npy"""


__UpperCAmelCase    				=							logging.get_logger(__name__)


def 					_snake_case   (     A ,    A       )						->    np.ndarray:
							import tensorflow.compat.va as tf

							lowerCAmelCase__          =	tf.data.TFRecordDataset(UpperCamelCase_ ,    buffer_size=512 * 1024 * 1024       )
							lowerCAmelCase__          =	blocks_dataset.batch(UpperCamelCase_ ,    drop_remainder=UpperCamelCase_       )
							lowerCAmelCase__          =	next(blocks_dataset.take(1       ).as_numpy_iterator()       )

							return np_record




class 		a__				:





							'''simple docstring'''






							def __init__(		self					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_=2					,    lowerCamelCase_=10_00					,    lowerCamelCase_=1_00					,    lowerCamelCase_=10_00_00					,    )       ->							str:
														from scann.scann_ops.py.scann_ops_pybind import builder as Builder

														lowerCAmelCase__          =	Builder(db=_UpperCamelCase					,    num_neighbors=_UpperCamelCase					,    distance_measure='''dot_product'''   )
														lowerCAmelCase__          =	builder.tree(
														    num_leaves=_UpperCamelCase					,    num_leaves_to_search=_UpperCamelCase					,    training_sample_size=_UpperCamelCase   )
														lowerCAmelCase__          =	builder.score_ah(dimensions_per_block=_UpperCamelCase   )

														lowerCAmelCase__          =	builder.build()







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_   )       ->							Any:
														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.searcher.search_batched(question_projection.detach().cpu()   )
														return retrieved_block_ids.astype('''int64'''   )




class 		a__				:





							'''simple docstring'''






							def __init__(		self					,    lowerCamelCase_					,    lowerCamelCase_   )       ->							str:
														super().__init__()
														lowerCAmelCase__          =	block_records
														lowerCAmelCase__          =	tokenizer







							def __call__(		self					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_=None					,    lowerCamelCase_="pt"   )       ->							Optional[int]:
														lowerCAmelCase__          =	np.take(self.block_records					,    indices=_UpperCamelCase					,    axis=0   )

														lowerCAmelCase__          =	self.tokenizer.decode(question_input_ids[0]					,    skip_special_tokens=_UpperCamelCase   )

														lowerCAmelCase__          =	[]
														lowerCAmelCase__          =	[]
														for retrieved_block in retrieved_blocks:
																					text.append(_UpperCamelCase   )
																					text_pair.append(retrieved_block.decode()   )

														lowerCAmelCase__          =	self.tokenizer(
														    _UpperCamelCase					,    _UpperCamelCase					,    padding=_UpperCamelCase					,    truncation=_UpperCamelCase					,    return_special_tokens_mask=_UpperCamelCase					,    max_length=_UpperCamelCase   )
														lowerCAmelCase__          =	concat_inputs.convert_to_tensors(_UpperCamelCase   )

														if answer_ids is not None:
																					return self.block_has_answer(_UpperCamelCase					,    _UpperCamelCase   ) + (concat_inputs_tensors,)
														else:
																					return (None, None, None, concat_inputs_tensors)







							@classmethod
							def   __SCREAMING_SNAKE_CASE							(		cls					,    lowerCamelCase_					,    *lowerCamelCase_					,    **lowerCamelCase_   )       ->							Union[str, Any]:
														if os.path.isdir(_UpperCamelCase   ):
																					lowerCAmelCase__          =	os.path.join(_UpperCamelCase					,    _REALM_BLOCK_RECORDS_FILENAME   )
														else:
																					lowerCAmelCase__          =	hf_hub_download(
																					    repo_id=_UpperCamelCase					,    filename=_REALM_BLOCK_RECORDS_FILENAME					,    **_UpperCamelCase   )
														lowerCAmelCase__          =	np.load(_UpperCamelCase					,    allow_pickle=_UpperCamelCase   )

														lowerCAmelCase__          =	AutoTokenizer.from_pretrained(_UpperCamelCase					,    *_UpperCamelCase					,    **_UpperCamelCase   )

														return cls(_UpperCamelCase					,    _UpperCamelCase   )







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_   )       ->							Tuple:
														np.save(os.path.join(_UpperCamelCase					,    _REALM_BLOCK_RECORDS_FILENAME   )					,    self.block_records   )
														# save tokenizer
														self.tokenizer.save_pretrained(_UpperCamelCase   )







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_					,    lowerCamelCase_   )       ->							Dict:
														lowerCAmelCase__          =	[]
														lowerCAmelCase__          =	[]
														lowerCAmelCase__          =	[]
														lowerCAmelCase__          =	0

														for input_id in concat_inputs.input_ids:
																					lowerCAmelCase__          =	input_id.tolist()
																					# Check answers between two [SEP] tokens
																					lowerCAmelCase__          =	input_id_list.index(self.tokenizer.sep_token_id   )
																					lowerCAmelCase__          =	first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id   )

																					start_pos.append([]   )
																					end_pos.append([]   )
																					for answer in answer_ids:
																												for idx in range(first_sep_idx + 1					,    _UpperCamelCase   ):
																																			if answer[0] == input_id_list[idx]:
																																										if input_id_list[idx : idx + len(_UpperCamelCase   )] == answer:
																																																	start_pos[-1].append(_UpperCamelCase   )
																																																	end_pos[-1].append(idx + len(_UpperCamelCase   ) - 1   )

																					if len(start_pos[-1]   ) == 0:
																												has_answers.append(_UpperCamelCase   )
																					else:
																												has_answers.append(_UpperCamelCase   )
																												if len(start_pos[-1]   ) > max_answers:
																																			lowerCAmelCase__          =	len(start_pos[-1]   )

        # Pad -1 to max_answers
														for start_pos_, end_pos_ in zip(_UpperCamelCase					,    _UpperCamelCase   ):
																					if len(start_pos_   ) < max_answers:
																												lowerCAmelCase__          =	[-1] * (max_answers - len(start_pos_   ))
																												start_pos_ += padded
																												end_pos_ += padded
														return has_answers, start_pos, end_pos







import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


__snake_case			:			Union[str, Any]			   =   logging.get_logger(__name__)


@dataclass
class 			__SCREAMING_SNAKE_CASE    :
						_SCREAMING_SNAKE_CASE		:				str
						_SCREAMING_SNAKE_CASE		:				str
						_SCREAMING_SNAKE_CASE		:				Optional[str]            =    None
						_SCREAMING_SNAKE_CASE		:				Optional[str]            =    None






						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return json.dumps(dataclasses.asdict(self      )   ,  indent=2      ) + "\n"




@dataclass(frozen=__lowercase)
class 			__SCREAMING_SNAKE_CASE    :
						_SCREAMING_SNAKE_CASE		:				List[int]
						_SCREAMING_SNAKE_CASE		:				Optional[List[int]]            =    None
						_SCREAMING_SNAKE_CASE		:				Optional[List[int]]            =    None
						_SCREAMING_SNAKE_CASE		:				Optional[Union[int, float]]            =    None






						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return json.dumps(dataclasses.asdict(self      )      ) + "\n"




class 			__SCREAMING_SNAKE_CASE    :






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												raise NotImplementedError()






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												raise NotImplementedError()






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												raise NotImplementedError()






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												raise NotImplementedError()






						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												raise NotImplementedError()






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												if len(self.get_labels()      ) > 1:
																		lowerCAmelCase__			  =  self.get_labels()[int(example.label      )]
												return example







						@classmethod
						def 					UpperCamelCase__					(		cls   ,  _UpperCamelCase   ,  _UpperCamelCase=None      ):






												"""simple docstring"""
												with open(_UpperCamelCase   ,  'r'   ,  encoding='utf-8-sig'      ) as f:
																		return list(csv.reader(_UpperCamelCase   ,  delimiter='\t'   ,  quotechar=_UpperCamelCase      )      )




class 			__SCREAMING_SNAKE_CASE    (      __lowercase):






						def __init__(		self   ,  _UpperCamelCase=None   ,  _UpperCamelCase=None   ,  _UpperCamelCase="classification"   ,  _UpperCamelCase=False      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  [] if labels is None else labels
												lowerCAmelCase__			  =  [] if examples is None else examples
												lowerCAmelCase__			  =  mode
												lowerCAmelCase__			  =  verbose






						def __len__(		self      ):






												"""simple docstring"""
												return len(self.examples      )






						def __getitem__(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												if isinstance(_UpperCamelCase   ,  _UpperCamelCase      ):
																		return SingleSentenceClassificationProcessor(labels=self.labels   ,  examples=self.examples[idx]      )
												return self.examples[idx]






						@classmethod
						def 					UpperCamelCase__					(		cls   ,  _UpperCamelCase   ,  _UpperCamelCase=""   ,  _UpperCamelCase=0   ,  _UpperCamelCase=1   ,  _UpperCamelCase=None   ,  _UpperCamelCase=False   ,  **_UpperCamelCase      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  cls(**_UpperCamelCase      )
												processor.add_examples_from_csv(
												    _UpperCamelCase   ,  split_name=_UpperCamelCase   ,  column_label=_UpperCamelCase   ,  column_text=_UpperCamelCase   ,  column_id=_UpperCamelCase   ,  skip_first_row=_UpperCamelCase   ,  overwrite_labels=_UpperCamelCase   ,  overwrite_examples=_UpperCamelCase   ,  )
												return processor






						@classmethod
						def 					UpperCamelCase__					(		cls   ,  _UpperCamelCase   ,  _UpperCamelCase=None   ,  **_UpperCamelCase      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  cls(**_UpperCamelCase      )
												processor.add_examples(_UpperCamelCase   ,  labels=_UpperCamelCase      )
												return processor






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase=""   ,  _UpperCamelCase=0   ,  _UpperCamelCase=1   ,  _UpperCamelCase=None   ,  _UpperCamelCase=False   ,  _UpperCamelCase=False   ,  _UpperCamelCase=False   ,  ):






												"""simple docstring"""
												lowerCAmelCase__			  =  self._read_tsv(_UpperCamelCase      )
												if skip_first_row:
																		lowerCAmelCase__			  =  lines[1:]
												lowerCAmelCase__			  =  []
												lowerCAmelCase__			  =  []
												lowerCAmelCase__			  =  []
												for i, line in enumerate(_UpperCamelCase      ):
																		texts.append(line[column_text]      )
																		labels.append(line[column_label]      )
																		if column_id is not None:
																								ids.append(line[column_id]      )
																		else:
																								lowerCAmelCase__			  =  F"{split_name}-{i}" if split_name else str(_UpperCamelCase      )
																								ids.append(_UpperCamelCase      )

												return self.add_examples(
												    _UpperCamelCase   ,  _UpperCamelCase   ,  _UpperCamelCase   ,  overwrite_labels=_UpperCamelCase   ,  overwrite_examples=_UpperCamelCase      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase=None   ,  _UpperCamelCase=None   ,  _UpperCamelCase=False   ,  _UpperCamelCase=False      ):






												"""simple docstring"""
												if labels is not None and len(_UpperCamelCase      ) != len(_UpperCamelCase      ):
																		raise ValueError(
																		    F"Text and labels have mismatched lengths {len(_UpperCamelCase      )} and {len(_UpperCamelCase      )}"      )
												if ids is not None and len(_UpperCamelCase      ) != len(_UpperCamelCase      ):
																		raise ValueError(F"Text and ids have mismatched lengths {len(_UpperCamelCase      )} and {len(_UpperCamelCase      )}"      )
												if ids is None:
																		lowerCAmelCase__			  =  [None] * len(_UpperCamelCase      )
												if labels is None:
																		lowerCAmelCase__			  =  [None] * len(_UpperCamelCase      )
												lowerCAmelCase__			  =  []
												lowerCAmelCase__			  =  set()
												for text_or_text_and_label, label, guid in zip(_UpperCamelCase   ,  _UpperCamelCase   ,  _UpperCamelCase      ):
																		if isinstance(_UpperCamelCase   ,  (tuple, list)      ) and label is None:
																								lowerCAmelCase__  ,				lowerCAmelCase__			  =  text_or_text_and_label
																		else:
																								lowerCAmelCase__			  =  text_or_text_and_label
																		added_labels.add(_UpperCamelCase      )
																		examples.append(InputExample(guid=_UpperCamelCase   ,  text_a=_UpperCamelCase   ,  text_b=_UpperCamelCase   ,  label=_UpperCamelCase      )      )

												# Update examples
												if overwrite_examples:
																		lowerCAmelCase__			  =  examples
												else:
																		self.examples.extend(_UpperCamelCase      )

												# Update labels
												if overwrite_labels:
																		lowerCAmelCase__			  =  list(_UpperCamelCase      )
												else:
																		lowerCAmelCase__			  =  list(set(self.labels      ).union(_UpperCamelCase      )      )

												return self.examples







						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase=None   ,  _UpperCamelCase=False   ,  _UpperCamelCase=0   ,  _UpperCamelCase=True   ,  _UpperCamelCase=None   ,  ):






												"""simple docstring"""
												if max_length is None:
																		lowerCAmelCase__			  =  tokenizer.max_len

												lowerCAmelCase__			  =  {label: i for i, label in enumerate(self.labels      )}

												lowerCAmelCase__			  =  []
												for ex_index, example in enumerate(self.examples      ):
																		if ex_index % 1_00_00 == 0:
																								logger.info(F"Tokenizing example {ex_index}"      )

																		lowerCAmelCase__			  =  tokenizer.encode(
																		    example.text_a   ,  add_special_tokens=_UpperCamelCase   ,  max_length=min(_UpperCamelCase   ,  tokenizer.max_len      )   ,  )
																		all_input_ids.append(_UpperCamelCase      )

												lowerCAmelCase__			  =  max(len(_UpperCamelCase      ) for input_ids in all_input_ids      )

												lowerCAmelCase__			  =  []
												for ex_index, (input_ids, example) in enumerate(zip(_UpperCamelCase   ,  self.examples      )      ):
																		if ex_index % 1_00_00 == 0:
																								logger.info(F"Writing example {ex_index}/{len(self.examples      )}"      )
																		# The mask has 1 for real tokens and 0 for padding tokens. Only real
																		# tokens are attended to.
																		lowerCAmelCase__			  =  [1 if mask_padding_with_zero else 0] * len(_UpperCamelCase      )

																		# Zero-pad up to the sequence length.
																		lowerCAmelCase__			  =  batch_length - len(_UpperCamelCase      )
																		if pad_on_left:
																								lowerCAmelCase__			  =  ([pad_token] * padding_length) + input_ids
																								lowerCAmelCase__			  =  ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
																		else:
																								lowerCAmelCase__			  =  input_ids + ([pad_token] * padding_length)
																								lowerCAmelCase__			  =  attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

																		if len(_UpperCamelCase      ) != batch_length:
																								raise ValueError(F"Error with input length {len(_UpperCamelCase      )} vs {batch_length}"      )
																		if len(_UpperCamelCase      ) != batch_length:
																								raise ValueError(F"Error with input length {len(_UpperCamelCase      )} vs {batch_length}"      )

																		if self.mode == "classification":
																								lowerCAmelCase__			  =  label_map[example.label]
																		elif self.mode == "regression":
																								lowerCAmelCase__			  =  float(example.label      )
																		else:
																								raise ValueError(self.mode      )

																		if ex_index < 5 and self.verbose:
																								logger.info('*** Example ***'      )
																								logger.info(F"guid: {example.guid}"      )
																								logger.info(F"input_ids: {' '.join([str(_UpperCamelCase      ) for x in input_ids]      )}"      )
																								logger.info(F"attention_mask: {' '.join([str(_UpperCamelCase      ) for x in attention_mask]      )}"      )
																								logger.info(F"label: {example.label} (id = {label})"      )

																		features.append(InputFeatures(input_ids=_UpperCamelCase   ,  attention_mask=_UpperCamelCase   ,  label=_UpperCamelCase      )      )

												if return_tensors is None:
																		return features
												elif return_tensors == "tf":
																		if not is_tf_available():
																								raise RuntimeError('return_tensors set to \'tf\' but TensorFlow 2.0 can\'t be imported'      )
																		import tensorflow as tf

																		def gen():
																								for ex in features:
																														yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

																		lowerCAmelCase__			  =  tf.data.Dataset.from_generator(
																		    _UpperCamelCase   ,  ({'input_ids': tf.intaa, 'attention_mask': tf.intaa}, tf.intaa)   ,  ({'input_ids': tf.TensorShape([None]      ), 'attention_mask': tf.TensorShape([None]      )}, tf.TensorShape([]      ))   ,  )
																		return dataset
												elif return_tensors == "pt":
																		if not is_torch_available():
																								raise RuntimeError('return_tensors set to \'pt\' but PyTorch can\'t be imported'      )
																		import torch
																		from torch.utils.data import TensorDataset

																		lowerCAmelCase__			  =  torch.tensor([f.input_ids for f in features]   ,  dtype=torch.long      )
																		lowerCAmelCase__			  =  torch.tensor([f.attention_mask for f in features]   ,  dtype=torch.long      )
																		if self.mode == "classification":
																								lowerCAmelCase__			  =  torch.tensor([f.label for f in features]   ,  dtype=torch.long      )
																		elif self.mode == "regression":
																								lowerCAmelCase__			  =  torch.tensor([f.label for f in features]   ,  dtype=torch.float      )

																		lowerCAmelCase__			  =  TensorDataset(_UpperCamelCase   ,  _UpperCamelCase   ,  _UpperCamelCase      )
																		return dataset
												else:
																		raise ValueError('return_tensors should be one of \'tf\' or \'pt\''      )





