


'''simple docstring'''

import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowerCAmelCase				:Union[str, Any]        =      2_0_4_8
lowerCAmelCase				:Dict        =      4_0_9_6
lowerCAmelCase				:Optional[Any]        =      4_2
lowerCAmelCase				:Dict        =      os.environ.pop('''PROCESS_TRAIN''', '''false''')
lowerCAmelCase				:Union[str, Any]        =      {'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}





def lowerCamelCase      (      lowerCAmelCase    :     Dict		):





	"""simple docstring"""







	def choose_first(lowerCAmelCase    :     Optional[int]		,		lowerCAmelCase    :     List[str]=False		):
		assert isinstance(lowerCAmelCase		,		lowerCAmelCase		)
		if len(lowerCAmelCase		) == 1:
			__magic_name__	:   Tuple					     =					answer[0]
			return {k: [answer[k]] for k in answer} if is_long_answer else answer
		for a in answer:
			if is_long_answer:
				__magic_name__	:   Optional[Any]					     =					{k: [a[k]] for k in a}
			if len(a['start_token']		) > 0:
				break
		return a

	__magic_name__	:   Tuple					     =					{'id': example['id']}
	__magic_name__	:   Optional[Any]					     =					example['annotations']
	__magic_name__	:   Optional[Any]					     =					annotation['yes_no_answer']
	if 0 in yes_no_answer or 1 in yes_no_answer:
		__magic_name__	:   str					     =					['yes'] if 1 in yes_no_answer else ['no']
		__magic_name__	:   int					     =					[]
		__magic_name__	:   int					     =					[]
		__magic_name__	:   List[str]					     =					['<cls>']
	else:
		__magic_name__	:   Tuple					     =					['short']
		__magic_name__	:   Optional[Any]					     =					choose_first(annotation['short_answers']		)
		if len(out['start_token']		) == 0:
			# answer will be long if short is not available
			__magic_name__	:   Any					     =					['long']
			__magic_name__	:   List[Any]					     =					choose_first(annotation['long_answer']		,		is_long_answer=lowerCAmelCase		)
			__magic_name__	:   Dict					     =					[]
		answer.update(lowerCAmelCase		)

	# disregard some samples
	if len(answer['start_token']		) > 1 or answer["start_token"] == answer["end_token"]:
		__magic_name__	:   str					     =					True
	else:
		__magic_name__	:   Optional[Any]					     =					False

	__magic_name__	:   Optional[int]					     =					['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
	if not all(isinstance(answer[k]		,		lowerCAmelCase		) for k in cols		):
		raise ValueError('Issue in ID'		,		example['id']		)

	return answer





def lowerCamelCase      (      lowerCAmelCase    :     Union[str, Any]		,		lowerCAmelCase    :     Any=False		):





	"""simple docstring"""







	__magic_name__	:   Optional[Any]					     =					_get_single_answer(lowerCAmelCase		)
	# bytes are of no use
	del answer["start_byte"]
	del answer["end_byte"]

	# handle yes_no answers explicitly
	if answer["category"][0] in ["yes", "no"]:  # category is list with one element
		__magic_name__	:   Any					     =					example['document']['tokens']
		__magic_name__	:   List[str]					     =					[]
		for i in range(len(doc['token']		)		):
			if not doc["is_html"][i]:
				context.append(doc['token'][i]		)
		return {
		    "context": " ".join(lowerCAmelCase		),
		    "answer": {
		        "start_token": -100,  # ignore index in cross-entropy
		        "end_token": -100,  # ignore index in cross-entropy
		        "category": answer["category"],
		        "span": answer["category"],  # extra
		    },
		}

	# later, help in removing all no answers
	if answer["start_token"] == [-1]:
		return {
		    "context": "None",
		    "answer": {
		        "start_token": -1,
		        "end_token": -1,
		        "category": "null",
		        "span": "None",  # extra
		    },
		}

	# handling normal samples

	__magic_name__	:   Optional[int]					     =					['start_token', 'end_token']
	answer.update({k: answer[k][0] if len(answer[k]		) > 0 else answer[k] for k in cols}		)  # e.g. [10] == 10

	__magic_name__	:   int					     =					example['document']['tokens']
	__magic_name__	:   Optional[Any]					     =					answer['start_token']
	__magic_name__	:   str					     =					answer['end_token']

	__magic_name__	:   Dict					     =					[]
	for i in range(len(doc['token']		)		):
		if not doc["is_html"][i]:
			context.append(doc['token'][i]		)
		else:
			if answer["start_token"] > i:
				start_token -= 1
			if answer["end_token"] > i:
				end_token -= 1
	__magic_name__	:   int					     =					' '.join(context[start_token:end_token]		)

	# checking above code
	if assertion:
		__magic_name__	:   List[str]					     =					doc['is_html'][answer['start_token'] : answer['end_token']]
		__magic_name__	:   str					     =					doc['token'][answer['start_token'] : answer['end_token']]
		__magic_name__	:   List[Any]					     =					' '.join([old[i] for i in range(len(lowerCAmelCase		)		) if not is_html[i]]		)
		if new != old:
			print('ID:'		,		example['id']		)
			print('New:'		,		lowerCAmelCase		,		end='\n'		)
			print('Old:'		,		lowerCAmelCase		,		end='\n\n'		)

	return {
	    "context": " ".join(lowerCAmelCase		),
	    "answer": {
	        "start_token": start_token,
	        "end_token": end_token - 1,  # this makes it inclusive
	        "category": answer["category"],  # either long or short
	        "span": new,  # extra
	    },
	}





def lowerCamelCase      (      lowerCAmelCase    :     Dict		,		lowerCAmelCase    :     List[str]		,		lowerCAmelCase    :     Tuple=2048		,		lowerCAmelCase    :     List[str]=4096		,		lowerCAmelCase    :     Optional[int]=True		):





	"""simple docstring"""







	__magic_name__	:   List[str]					     =					get_context_and_ans(lowerCAmelCase		,		assertion=lowerCAmelCase		)
	__magic_name__	:   Union[str, Any]					     =					out['answer']

	# later, removing these samples
	if answer["start_token"] == -1:
		return {
		    "example_id": example["id"],
		    "input_ids": [[-1]],
		    "labels": {
		        "start_token": [-1],
		        "end_token": [-1],
		        "category": ["null"],
		    },
		}

	__magic_name__	:   Optional[int]					     =					tokenizer(example['question']['text']		,		out['context']		).input_ids
	__magic_name__	:   List[str]					     =					input_ids.index(tokenizer.sep_token_id		) + 1

	# return yes/no
	if answer["category"][0] in ["yes", "no"]:  # category is list with one element
		__magic_name__	:   Tuple					     =					[]
		__magic_name__	:   Optional[int]					     =					[]
		__magic_name__	:   Tuple					     =					input_ids[:q_len]
		__magic_name__	:   List[Any]					     =					range(lowerCAmelCase		,		len(lowerCAmelCase		)		,		max_length - doc_stride		)
		for i in doc_start_indices:
			__magic_name__	:   Optional[int]					     =					i + max_length - q_len
			__magic_name__	:   List[str]					     =					input_ids[i:end_index]
			inputs.append(q_indices + slice		)
			category.append(answer['category'][0]		)
			if slice[-1] == tokenizer.sep_token_id:
				break

		return {
		    "example_id": example["id"],
		    "input_ids": inputs,
		    "labels": {
		        "start_token": [-100] * len(lowerCAmelCase		),
		        "end_token": [-100] * len(lowerCAmelCase		),
		        "category": category,
		    },
		}

	__magic_name__	:   List[Any]					     =					out['context'].split()
	__magic_name__	:   Optional[int]					     =					splitted_context[answer['end_token']]
	__magic_name__	:   List[Any]					     =					len(
	    tokenizer(
	        ' '.join(splitted_context[: answer['start_token']]		)		,		add_special_tokens=lowerCAmelCase		,		).input_ids		)
	__magic_name__	:   List[Any]					     =					len(
	    tokenizer(' '.join(splitted_context[: answer['end_token']]		)		,		add_special_tokens=lowerCAmelCase		).input_ids		)

	answer["start_token"] += q_len
	answer["end_token"] += q_len

	# fixing end token
	__magic_name__	:   Tuple					     =					len(tokenizer(lowerCAmelCase		,		add_special_tokens=lowerCAmelCase		).input_ids		)
	if num_sub_tokens > 1:
		answer["end_token"] += num_sub_tokens - 1

	__magic_name__	:   Any					     =					input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
	__magic_name__	:   Union[str, Any]					     =					answer['start_token']
	__magic_name__	:   Optional[Any]					     =					answer['end_token']

	if assertion:
		__magic_name__	:   int					     =					tokenizer.decode(lowerCAmelCase		)
		if answer["span"] != new:
			print('ISSUE IN TOKENIZATION'		)
			print('OLD:'		,		answer['span']		)
			print('NEW:'		,		lowerCAmelCase		,		end='\n\n'		)

	if len(lowerCAmelCase		) <= max_length:
		return {
		    "example_id": example["id"],
		    "input_ids": [input_ids],
		    "labels": {
		        "start_token": [answer["start_token"]],
		        "end_token": [answer["end_token"]],
		        "category": answer["category"],
		    },
		}

	__magic_name__	:   Union[str, Any]					     =					input_ids[:q_len]
	__magic_name__	:   int					     =					range(lowerCAmelCase		,		len(lowerCAmelCase		)		,		max_length - doc_stride		)

	__magic_name__	:   Dict					     =					[]
	__magic_name__	:   Optional[int]					     =					[]
	__magic_name__	:   List[str]					     =					[]
	__magic_name__	:   int					     =					[]  # null, yes, no, long, short
	for i in doc_start_indices:
		__magic_name__	:   Tuple					     =					i + max_length - q_len
		__magic_name__	:   List[Any]					     =					input_ids[i:end_index]
		inputs.append(q_indices + slice		)
		assert len(inputs[-1]		) <= max_length, "Issue in truncating length"

		if start_token >= i and end_token <= end_index - 1:
			__magic_name__	:   Union[str, Any]					     =					start_token - i + q_len
			__magic_name__	:   Optional[int]					     =					end_token - i + q_len
			answers_category.append(answer['category'][0]		)  # ["short"] -> "short"
		else:
			__magic_name__	:   Union[str, Any]					     =					-100
			__magic_name__	:   Dict					     =					-100
			answers_category.append('null'		)
		__magic_name__	:   List[str]					     =					inputs[-1][start_token : end_token + 1]

		answers_start_token.append(lowerCAmelCase		)
		answers_end_token.append(lowerCAmelCase		)
		if assertion:
			if new != old and new != [tokenizer.cls_token_id]:
				print('ISSUE in strided for ID:'		,		example['id']		)
				print('New:'		,		tokenizer.decode(lowerCAmelCase		)		)
				print('Old:'		,		tokenizer.decode(lowerCAmelCase		)		,		end='\n\n'		)
		if slice[-1] == tokenizer.sep_token_id:
			break

	return {
	    "example_id": example["id"],
	    "input_ids": inputs,
	    "labels": {
	        "start_token": answers_start_token,
	        "end_token": answers_end_token,
	        "category": answers_category,
	    },
	}





def lowerCamelCase      (      lowerCAmelCase    :     str		,		lowerCAmelCase    :     int		,		lowerCAmelCase    :     List[Any]=2048		,		lowerCAmelCase    :     Union[str, Any]=4096		,		lowerCAmelCase    :     List[Any]=False		):





	"""simple docstring"""







	__magic_name__	:   Union[str, Any]					     =					get_strided_contexts_and_ans(
	    lowerCAmelCase		,		lowerCAmelCase		,		doc_stride=lowerCAmelCase		,		max_length=lowerCAmelCase		,		assertion=lowerCAmelCase		,		)

	return example





def lowerCamelCase      (      lowerCAmelCase    :     Tuple		,		lowerCAmelCase    :     Tuple		):





	"""simple docstring"""







	with jsonlines.open(lowerCAmelCase		,		'a'		) as writer:
		for example in tqdm(lowerCAmelCase		,		total=len(lowerCAmelCase		)		,		desc='Saving samples ... '		):
			__magic_name__	:   Union[str, Any]					     =					example['labels']
			for ids, start, end, cat in zip(
			    example['input_ids']		,		labels['start_token']		,		labels['end_token']		,		labels['category']		,		):
				if start == -1 and end == -1:
					continue  # leave waste samples with no answer
				if cat == "null" and np.random.rand() < 0.6:
					continue  # removing 50 % samples
				writer.write(
				    {
				        'input_ids': ids,
				        'start_token': start,
				        'end_token': end,
				        'category': CATEGORY_MAPPING[cat],
				    }		)


if __name__ == "__main__":
							from datasets import load_dataset

							from transformers import BigBirdTokenizer

							lowerCAmelCase				:List[str]        =      load_dataset('''natural_questions''')
							lowerCAmelCase				:str        =      BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

							lowerCAmelCase				:List[Any]        =      data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

							lowerCAmelCase				:Any        =      {
							    '''tokenizer''': tokenizer,
							    '''doc_stride''': DOC_STRIDE,
							    '''max_length''': MAX_LENGTH,
							    '''assertion''': False,
							}
							lowerCAmelCase				:List[str]        =      data.map(prepare_inputs, fn_kwargs=fn_kwargs)
							lowerCAmelCase				:Dict        =      data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
							print(data)

							np.random.seed(SEED)
							lowerCAmelCase				:Any        =      '''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
							save_to_disk(data, file_name=cache_file_name)



'''simple docstring'''

# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ..models.auto import AutoModelForVisionaSeq
from ..utils import requires_backends
from .base import PipelineTool


if TYPE_CHECKING:
							from PIL import Image
class _lowerCamelCase	(						lowercase__							):



		'''simple docstring'''







		A_     :      Any				   =    """Salesforce/blip-image-captioning-base"""
		A_     :      Any				   =    (
		    """This is a tool that generates a description of an image. It takes an input named `image` which should be the """
		    """image to caption, and returns a text that contains the description in English."""
		)
		A_     :      List[str]				   =    """image_captioner"""
		A_     :      Optional[Any]				   =    AutoModelForVisionaSeq

		A_     :      Any				   =    ["""image"""]
		A_     :      Union[str, Any]				   =    ["""text"""]







		def __init__( self		:						Dict  ,					*_A		:						Dict  ,					**_A		:						Any      )  ->			Dict:
			requires_backends(self  ,					['vision']      )
			super().__init__(*_A  ,					**_A      )







		def        __lowerCAmelCase		( self		:						Tuple  ,					_A		:						"Image"      )  ->			Union[str, Any]:
			return self.pre_processor(images=_A  ,					return_tensors='pt'      )







		def        __lowerCAmelCase		( self		:						Optional[Any]  ,					_A		:						str      )  ->			List[Any]:
			return self.model.generate(**_A      )







		def        __lowerCAmelCase		( self		:						int  ,					_A		:						Optional[int]      )  ->			Union[str, Any]:
			return self.pre_processor.batch_decode(_A  ,					skip_special_tokens=_A      )[0].strip()