
'''simple docstring'''







import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowercase       :				Tuple =   2048
lowercase       :				Dict =   4096
lowercase       :				Optional[Any] =   42
lowercase       :				List[str] =   os.environ.pop("PROCESS_TRAIN", "false")
lowercase       :				Optional[int] =   {"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}







def 							SCREAMING_SNAKE_CASE__      (  __A      )			->      Any:




	def choose_first(__A     ,							__A=False      ):
		assert isinstance(__A     ,							__A      )
		if len(__A      ) == 1:
			_snake_case  		=						answer[0]
			return {k: [answer[k]] for k in answer} if is_long_answer else answer
		for a in answer:
			if is_long_answer:
				_snake_case  		=						{k: [a[k]] for k in a}
			if len(a['start_token']      ) > 0:
				break
		return a

	_snake_case  		=						{'id': example['id']}
	_snake_case  		=						example['annotations']
	_snake_case  		=						annotation['yes_no_answer']
	if 0 in yes_no_answer or 1 in yes_no_answer:
		_snake_case  		=						['yes'] if 1 in yes_no_answer else ['no']
		_snake_case  		=						_snake_case  		=						[]
		_snake_case  		=						_snake_case  		=						[]
		_snake_case  		=						['<cls>']
	else:
		_snake_case  		=						['short']
		_snake_case  		=						choose_first(annotation['short_answers']      )
		if len(out['start_token']      ) == 0:
			# answer will be long if short is not available
			_snake_case  		=						['long']
			_snake_case  		=						choose_first(annotation['long_answer']     ,							is_long_answer=__A      )
			_snake_case  		=						[]
		answer.update(__A      )

	# disregard some samples
	if len(answer['start_token']      ) > 1 or answer["start_token"] == answer["end_token"]:
		_snake_case  		=						True
	else:
		_snake_case  		=						False

	_snake_case  		=						['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
	if not all(isinstance(answer[k]     ,							__A      ) for k in cols      ):
		raise ValueError('Issue in ID'     ,							example['id']      )

	return answer







def 							SCREAMING_SNAKE_CASE__      (  __A     ,							__A=False      )			->      List[str]:
	_snake_case  		=						_get_single_answer(__A      )
	# bytes are of no use
	del answer["start_byte"]
	del answer["end_byte"]

	# handle yes_no answers explicitly
	if answer["category"][0] in ["yes", "no"]:  # category is list with one element
		_snake_case  		=						example['document']['tokens']
		_snake_case  		=						[]
		for i in range(len(doc['token']      )      ):
			if not doc["is_html"][i]:
				context.append(doc['token'][i]      )
		return {
		    "context": " ".join(__A      ),
		    "answer": {
		        "start_token": -100,  # ignore index in cross-entropy
		        "end_token": -100,  # ignore index in cross-entropy
		        "category": answer["category"],
		        "span": answer["category"],  # extra
		    },
		}

	# later, help in removing all no answers
	if answer["start_token"] == [-1]:
		return {
		    "context": "None",
		    "answer": {
		        "start_token": -1,
		        "end_token": -1,
		        "category": "null",
		        "span": "None",  # extra
		    },
		}

	# handling normal samples

	_snake_case  		=						['start_token', 'end_token']
	answer.update({k: answer[k][0] if len(answer[k]      ) > 0 else answer[k] for k in cols}      )  # e.g. [10] == 10

	_snake_case  		=						example['document']['tokens']
	_snake_case  		=						answer['start_token']
	_snake_case  		=						answer['end_token']

	_snake_case  		=						[]
	for i in range(len(doc['token']      )      ):
		if not doc["is_html"][i]:
			context.append(doc['token'][i]      )
		else:
			if answer["start_token"] > i:
				start_token -= 1
			if answer["end_token"] > i:
				end_token -= 1
	_snake_case  		=						' '.join(context[start_token:end_token]      )

	# checking above code
	if assertion:
		_snake_case  		=						doc['is_html'][answer['start_token'] : answer['end_token']]
		_snake_case  		=						doc['token'][answer['start_token'] : answer['end_token']]
		_snake_case  		=						' '.join([old[i] for i in range(len(__A      )      ) if not is_html[i]]      )
		if new != old:
			print('ID:'     ,							example['id']      )
			print('New:'     ,							__A     ,							end='\n'      )
			print('Old:'     ,							__A     ,							end='\n\n'      )

	return {
	    "context": " ".join(__A      ),
	    "answer": {
	        "start_token": start_token,
	        "end_token": end_token - 1,  # this makes it inclusive
	        "category": answer["category"],  # either long or short
	        "span": new,  # extra
	    },
	}







def 							SCREAMING_SNAKE_CASE__      (  __A     ,							__A     ,							__A=2_048     ,							__A=4_096     ,							__A=True      )			->      List[str]:
	_snake_case  		=						get_context_and_ans(__A     ,							assertion=__A      )
	_snake_case  		=						out['answer']

	# later, removing these samples
	if answer["start_token"] == -1:
		return {
		    "example_id": example["id"],
		    "input_ids": [[-1]],
		    "labels": {
		        "start_token": [-1],
		        "end_token": [-1],
		        "category": ["null"],
		    },
		}

	_snake_case  		=						tokenizer(example['question']['text']     ,							out['context']      ).input_ids
	_snake_case  		=						input_ids.index(tokenizer.sep_token_id      ) + 1

	# return yes/no
	if answer["category"][0] in ["yes", "no"]:  # category is list with one element
		_snake_case  		=						[]
		_snake_case  		=						[]
		_snake_case  		=						input_ids[:q_len]
		_snake_case  		=						range(__A     ,							len(__A      )     ,							max_length - doc_stride      )
		for i in doc_start_indices:
			_snake_case  		=						i + max_length - q_len
			_snake_case  		=						input_ids[i:end_index]
			inputs.append(q_indices + slice      )
			category.append(answer['category'][0]      )
			if slice[-1] == tokenizer.sep_token_id:
				break

		return {
		    "example_id": example["id"],
		    "input_ids": inputs,
		    "labels": {
		        "start_token": [-100] * len(__A      ),
		        "end_token": [-100] * len(__A      ),
		        "category": category,
		    },
		}

	_snake_case  		=						out['context'].split()
	_snake_case  		=						splitted_context[answer['end_token']]
	_snake_case  		=						len(
	    tokenizer(
	        ' '.join(splitted_context[: answer['start_token']]      )     ,							add_special_tokens=__A     ,							).input_ids      )
	_snake_case  		=						len(
	    tokenizer(' '.join(splitted_context[: answer['end_token']]      )     ,							add_special_tokens=__A      ).input_ids      )

	answer["start_token"] += q_len
	answer["end_token"] += q_len

	# fixing end token
	_snake_case  		=						len(tokenizer(__A     ,							add_special_tokens=__A      ).input_ids      )
	if num_sub_tokens > 1:
		answer["end_token"] += num_sub_tokens - 1

	_snake_case  		=						input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
	_snake_case  		=						answer['start_token']
	_snake_case  		=						answer['end_token']

	if assertion:
		_snake_case  		=						tokenizer.decode(__A      )
		if answer["span"] != new:
			print('ISSUE IN TOKENIZATION'      )
			print('OLD:'     ,							answer['span']      )
			print('NEW:'     ,							__A     ,							end='\n\n'      )

	if len(__A      ) <= max_length:
		return {
		    "example_id": example["id"],
		    "input_ids": [input_ids],
		    "labels": {
		        "start_token": [answer["start_token"]],
		        "end_token": [answer["end_token"]],
		        "category": answer["category"],
		    },
		}

	_snake_case  		=						input_ids[:q_len]
	_snake_case  		=						range(__A     ,							len(__A      )     ,							max_length - doc_stride      )

	_snake_case  		=						[]
	_snake_case  		=						[]
	_snake_case  		=						[]
	_snake_case  		=						[]  # null, yes, no, long, short
	for i in doc_start_indices:
		_snake_case  		=						i + max_length - q_len
		_snake_case  		=						input_ids[i:end_index]
		inputs.append(q_indices + slice      )
		assert len(inputs[-1]      ) <= max_length, "Issue in truncating length"

		if start_token >= i and end_token <= end_index - 1:
			_snake_case  		=						start_token - i + q_len
			_snake_case  		=						end_token - i + q_len
			answers_category.append(answer['category'][0]      )  # ["short"] -> "short"
		else:
			_snake_case  		=						-100
			_snake_case  		=						-100
			answers_category.append('null'      )
		_snake_case  		=						inputs[-1][start_token : end_token + 1]

		answers_start_token.append(__A      )
		answers_end_token.append(__A      )
		if assertion:
			if new != old and new != [tokenizer.cls_token_id]:
				print('ISSUE in strided for ID:'     ,							example['id']      )
				print('New:'     ,							tokenizer.decode(__A      )      )
				print('Old:'     ,							tokenizer.decode(__A      )     ,							end='\n\n'      )
		if slice[-1] == tokenizer.sep_token_id:
			break

	return {
	    "example_id": example["id"],
	    "input_ids": inputs,
	    "labels": {
	        "start_token": answers_start_token,
	        "end_token": answers_end_token,
	        "category": answers_category,
	    },
	}







def 							SCREAMING_SNAKE_CASE__      (  __A     ,							__A     ,							__A=2_048     ,							__A=4_096     ,							__A=False      )			->      Tuple:
	_snake_case  		=						get_strided_contexts_and_ans(
	    __A     ,							__A     ,							doc_stride=__A     ,							max_length=__A     ,							assertion=__A     ,							)

	return example







def 							SCREAMING_SNAKE_CASE__      (  __A     ,							__A      )			->      Union[str, Any]:




	with jsonlines.open(__A     ,							'a'      ) as writer:
		for example in tqdm(__A     ,							total=len(__A      )     ,							desc='Saving samples ... '      ):
			_snake_case  		=						example['labels']
			for ids, start, end, cat in zip(
			    example['input_ids']     ,							labels['start_token']     ,							labels['end_token']     ,							labels['category']     ,							):
				if start == -1 and end == -1:
					continue  # leave waste samples with no answer
				if cat == "null" and np.random.rand() < 0.6:
					continue  # removing 50 % samples
				writer.write(
				    {
				        'input_ids': ids,
				        'start_token': start,
				        'end_token': end,
				        'category': CATEGORY_MAPPING[cat],
				    }      )


if __name__ == "__main__":
					from datasets import load_dataset

					from transformers import BigBirdTokenizer

					lowercase       :				Dict =   load_dataset("natural_questions")
					lowercase       :				Union[str, Any] =   BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

					lowercase       :				int =   data["train" if PROCESS_TRAIN == "true" else "validation"]

					lowercase       :				Any =   {
					    "tokenizer": tokenizer,
					    "doc_stride": DOC_STRIDE,
					    "max_length": MAX_LENGTH,
					    "assertion": False,
					}
					lowercase       :				Any =   data.map(prepare_inputs, fn_kwargs=fn_kwargs)
					lowercase       :				Union[str, Any] =   data.remove_columns(["annotations", "document", "id", "question"])
					print(data)

					np.random.seed(SEED)
					lowercase       :				Tuple =   "nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
					save_to_disk(data, file_name=cache_file_name)








"""simple docstring"""





import logging
import os

from .state import PartialState

class 						lowercase   (   logging.LoggerAdapter   ):





      @staticmethod
      def        _snake_case     (			lowercase					)						->	Optional[Any]:
            lowerCAmelCase    		=			PartialState()
            return not main_process_only or (main_process_only and state.is_main_process)





      def        _snake_case     (			self ,		lowercase ,		lowercase ,		*lowercase ,		**lowercase					)						->	Dict:
            if PartialState._shared_state == {}:
                  raise RuntimeError(
                      """You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility."""					)
            lowerCAmelCase    		=			kwargs.pop("""main_process_only""" ,		lowercase					)
            lowerCAmelCase    		=			kwargs.pop("""in_order""" ,		lowercase					)

            if self.isEnabledFor(lowercase					):
                  if self._should_log(lowercase					):
                        lowerCAmelCase		,						lowerCAmelCase    		=			self.process(lowercase ,		lowercase					)
                        self.logger.log(lowercase ,		lowercase ,		*lowercase ,		**lowercase					)

                  elif in_order:
                        lowerCAmelCase    		=			PartialState()
                        for i in range(state.num_processes					):
                              if i == state.process_index:
                                    lowerCAmelCase		,						lowerCAmelCase    		=			self.process(lowercase ,		lowercase					)
                                    self.logger.log(lowercase ,		lowercase ,		*lowercase ,		**lowercase					)
                              state.wait_for_everyone()







def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		str				,		SCREAMING_SNAKE_CASE	:		str = None     ):







      '''simple docstring'''




      if log_level is None:
            lowerCAmelCase    		=			os.environ.get("""ACCELERATE_LOG_LEVEL"""				,		SCREAMING_SNAKE_CASE     )
      lowerCAmelCase    		=			logging.getLogger(SCREAMING_SNAKE_CASE     )
      if log_level is not None:
            logger.setLevel(log_level.upper()     )
            logger.root.setLevel(log_level.upper()     )
      return MultiProcessAdapter(SCREAMING_SNAKE_CASE				,		{}     )






