





import os

import jsonlines
import numpy as np
from tqdm import tqdm


UpperCAmelCase_							=				2_048
UpperCAmelCase_							=				4_096
UpperCAmelCase_							=				42
UpperCAmelCase_							=				os.environ.pop("""PROCESS_TRAIN""", """false""")
UpperCAmelCase_							=				{'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}





def  __magic_name__  (				lowercase				)				-> List[Any]:

							"""simple docstring"""



							def choose_first(lowercase				,       lowercase=False				):
														assert isinstance(_lowerCamelCase				,       _lowerCamelCase				)
														if len(_lowerCamelCase				) == 1:
																					lowercase_	:   Dict  				=     answer[0]
																					return {k: [answer[k]] for k in answer} if is_long_answer else answer
														for a in answer:
																					if is_long_answer:
																												lowercase_	:   Optional[Any]  				=     {k: [a[k]] for k in a}
																					if len(a["""start_token"""]				) > 0:
																												break
														return a

							lowercase_	:   Optional[Any]  				=     {"""id""": example["""id"""]}
							lowercase_	:   Dict  				=     example["""annotations"""]
							lowercase_	:   Optional[int]  				=     annotation["""yes_no_answer"""]
							if 0 in yes_no_answer or 1 in yes_no_answer:
														lowercase_	:   Tuple  				=     ["""yes"""] if 1 in yes_no_answer else ["""no"""]
														lowercase_	:   Optional[int]  				=     []
														lowercase_	:   List[Any]  				=     []
														lowercase_	:   int  				=     ["""<cls>"""]
							else:
														lowercase_	:   Dict  				=     ["""short"""]
														lowercase_	:   Optional[int]  				=     choose_first(annotation["""short_answers"""]				)
														if len(out["""start_token"""]				) == 0:
																					# answer will be long if short is not available
																					lowercase_	:   Tuple  				=     ["""long"""]
																					lowercase_	:   List[Any]  				=     choose_first(annotation["""long_answer"""]				,       is_long_answer=_lowerCamelCase				)
																					lowercase_	:   List[Any]  				=     []
														answer.update(_lowerCamelCase				)

							# disregard some samples
							if len(answer["""start_token"""]				) > 1 or answer["start_token"] == answer["end_token"]:
														lowercase_	:   Any  				=     True
							else:
														lowercase_	:   List[str]  				=     False

							lowercase_	:   List[Any]  				=     ["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
							if not all(isinstance(answer[k]				,       _lowerCamelCase				) for k in cols				):
														raise ValueError("""Issue in ID"""				,       example["""id"""]				)

							return answer





def  __magic_name__  (				lowercase				,       lowercase=False				)				-> Any:

							"""simple docstring"""



							lowercase_	:   int  				=     _get_single_answer(_lowerCamelCase				)
							# bytes are of no use
							del answer["start_byte"]
							del answer["end_byte"]

							# handle yes_no answers explicitly
							if answer["category"][0] in ["yes", "no"]:  # category is list with one element
														lowercase_	:   str  				=     example["""document"""]["""tokens"""]
														lowercase_	:   List[str]  				=     []
														for i in range(len(doc["""token"""]				)				):
																					if not doc["is_html"][i]:
																												context.append(doc["""token"""][i]				)
														return {
														    "context": " ".join(_lowerCamelCase				),
														    "answer": {
														        "start_token": -100,  # ignore index in cross-entropy
														        "end_token": -100,  # ignore index in cross-entropy
														        "category": answer["category"],
														        "span": answer["category"],  # extra
														    },
														}

							# later, help in removing all no answers
							if answer["start_token"] == [-1]:
														return {
														    "context": "None",
														    "answer": {
														        "start_token": -1,
														        "end_token": -1,
														        "category": "null",
														        "span": "None",  # extra
														    },
														}

							# handling normal samples

							lowercase_	:   Union[str, Any]  				=     ["""start_token""", """end_token"""]
							answer.update({k: answer[k][0] if len(answer[k]				) > 0 else answer[k] for k in cols}				)  # e.g. [10] == 10

							lowercase_	:   Any  				=     example["""document"""]["""tokens"""]
							lowercase_	:   int  				=     answer["""start_token"""]
							lowercase_	:   Optional[int]  				=     answer["""end_token"""]

							lowercase_	:   int  				=     []
							for i in range(len(doc["""token"""]				)				):
														if not doc["is_html"][i]:
																					context.append(doc["""token"""][i]				)
														else:
																					if answer["start_token"] > i:
																												start_token -= 1
																					if answer["end_token"] > i:
																												end_token -= 1
							lowercase_	:   Dict  				=     """ """.join(context[start_token:end_token]				)

							# checking above code
							if assertion:
														lowercase_	:   Any  				=     doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
														lowercase_	:   List[Any]  				=     doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
														lowercase_	:   str  				=     """ """.join([old[i] for i in range(len(_lowerCamelCase				)				) if not is_html[i]]				)
														if new != old:
																					print("""ID:"""				,       example["""id"""]				)
																					print("""New:"""				,       _lowerCamelCase				,       end="""\n"""				)
																					print("""Old:"""				,       _lowerCamelCase				,       end="""\n\n"""				)

							return {
							    "context": " ".join(_lowerCamelCase				),
							    "answer": {
							        "start_token": start_token,
							        "end_token": end_token - 1,  # this makes it inclusive
							        "category": answer["category"],  # either long or short
							        "span": new,  # extra
							    },
							}





def  __magic_name__  (				lowercase				,       lowercase				,       lowercase=2048				,       lowercase=4096				,       lowercase=True				)				-> List[Any]:

							"""simple docstring"""



							lowercase_	:   Tuple  				=     get_context_and_ans(_lowerCamelCase				,       assertion=_lowerCamelCase				)
							lowercase_	:   int  				=     out["""answer"""]

							# later, removing these samples
							if answer["start_token"] == -1:
														return {
														    "example_id": example["id"],
														    "input_ids": [[-1]],
														    "labels": {
														        "start_token": [-1],
														        "end_token": [-1],
														        "category": ["null"],
														    },
														}

							lowercase_	:   int  				=     tokenizer(example["""question"""]["""text"""]				,       out["""context"""]				).input_ids
							lowercase_	:   str  				=     input_ids.index(tokenizer.sep_token_id				) + 1

							# return yes/no
							if answer["category"][0] in ["yes", "no"]:  # category is list with one element
														lowercase_	:   Any  				=     []
														lowercase_	:   Optional[int]  				=     []
														lowercase_	:   int  				=     input_ids[:q_len]
														lowercase_	:   Tuple  				=     range(_lowerCamelCase				,       len(_lowerCamelCase				)				,       max_length - doc_stride				)
														for i in doc_start_indices:
																					lowercase_	:   Union[str, Any]  				=     i + max_length - q_len
																					lowercase_	:   str  				=     input_ids[i:end_index]
																					inputs.append(q_indices + slice				)
																					category.append(answer["""category"""][0]				)
																					if slice[-1] == tokenizer.sep_token_id:
																												break

														return {
														    "example_id": example["id"],
														    "input_ids": inputs,
														    "labels": {
														        "start_token": [-100] * len(_lowerCamelCase				),
														        "end_token": [-100] * len(_lowerCamelCase				),
														        "category": category,
														    },
														}

							lowercase_	:   str  				=     out["""context"""].split()
							lowercase_	:   Dict  				=     splitted_context[answer["""end_token"""]]
							lowercase_	:   Union[str, Any]  				=     len(
							    tokenizer(
							        """ """.join(splitted_context[: answer["""start_token"""]]				)				,       add_special_tokens=_lowerCamelCase				,       ).input_ids				)
							lowercase_	:   Union[str, Any]  				=     len(
							    tokenizer(""" """.join(splitted_context[: answer["""end_token"""]]				)				,       add_special_tokens=_lowerCamelCase				).input_ids				)

							answer["start_token"] += q_len
							answer["end_token"] += q_len

							# fixing end token
							lowercase_	:   int  				=     len(tokenizer(_lowerCamelCase				,       add_special_tokens=_lowerCamelCase				).input_ids				)
							if num_sub_tokens > 1:
														answer["end_token"] += num_sub_tokens - 1

							lowercase_	:   Union[str, Any]  				=     input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
							lowercase_	:   Dict  				=     answer["""start_token"""]
							lowercase_	:   Any  				=     answer["""end_token"""]

							if assertion:
														lowercase_	:   str  				=     tokenizer.decode(_lowerCamelCase				)
														if answer["span"] != new:
																					print("""ISSUE IN TOKENIZATION"""				)
																					print("""OLD:"""				,       answer["""span"""]				)
																					print("""NEW:"""				,       _lowerCamelCase				,       end="""\n\n"""				)

							if len(_lowerCamelCase				) <= max_length:
														return {
														    "example_id": example["id"],
														    "input_ids": [input_ids],
														    "labels": {
														        "start_token": [answer["start_token"]],
														        "end_token": [answer["end_token"]],
														        "category": answer["category"],
														    },
														}

							lowercase_	:   Dict  				=     input_ids[:q_len]
							lowercase_	:   List[Any]  				=     range(_lowerCamelCase				,       len(_lowerCamelCase				)				,       max_length - doc_stride				)

							lowercase_	:   Dict  				=     []
							lowercase_	:   Optional[Any]  				=     []
							lowercase_	:   List[Any]  				=     []
							lowercase_	:   Tuple  				=     []  # null, yes, no, long, short
							for i in doc_start_indices:
														lowercase_	:   Any  				=     i + max_length - q_len
														lowercase_	:   Any  				=     input_ids[i:end_index]
														inputs.append(q_indices + slice				)
														assert len(inputs[-1]				) <= max_length, "Issue in truncating length"

														if start_token >= i and end_token <= end_index - 1:
																					lowercase_	:   Tuple  				=     start_token - i + q_len
																					lowercase_	:   List[str]  				=     end_token - i + q_len
																					answers_category.append(answer["""category"""][0]				)  # ["short"] -> "short"
														else:
																					lowercase_	:   str  				=     -100
																					lowercase_	:   Union[str, Any]  				=     -100
																					answers_category.append("""null"""				)
														lowercase_	:   str  				=     inputs[-1][start_token : end_token + 1]

														answers_start_token.append(_lowerCamelCase				)
														answers_end_token.append(_lowerCamelCase				)
														if assertion:
																					if new != old and new != [tokenizer.cls_token_id]:
																												print("""ISSUE in strided for ID:"""				,       example["""id"""]				)
																												print("""New:"""				,       tokenizer.decode(_lowerCamelCase				)				)
																												print("""Old:"""				,       tokenizer.decode(_lowerCamelCase				)				,       end="""\n\n"""				)
														if slice[-1] == tokenizer.sep_token_id:
																					break

							return {
							    "example_id": example["id"],
							    "input_ids": inputs,
							    "labels": {
							        "start_token": answers_start_token,
							        "end_token": answers_end_token,
							        "category": answers_category,
							    },
							}





def  __magic_name__  (				lowercase				,       lowercase				,       lowercase=2048				,       lowercase=4096				,       lowercase=False				)				-> Any:

							"""simple docstring"""



							lowercase_	:   List[str]  				=     get_strided_contexts_and_ans(
							    _lowerCamelCase				,       _lowerCamelCase				,       doc_stride=_lowerCamelCase				,       max_length=_lowerCamelCase				,       assertion=_lowerCamelCase				,       )

							return example





def  __magic_name__  (				lowercase				,       lowercase				)				-> int:

							"""simple docstring"""



							with jsonlines.open(_lowerCamelCase				,       """a"""				) as writer:
														for example in tqdm(_lowerCamelCase				,       total=len(_lowerCamelCase				)				,       desc="""Saving samples ... """				):
																					lowercase_	:   Tuple  				=     example["""labels"""]
																					for ids, start, end, cat in zip(
																					    example["""input_ids"""]				,       labels["""start_token"""]				,       labels["""end_token"""]				,       labels["""category"""]				,       ):
																												if start == -1 and end == -1:
																																			continue  # leave waste samples with no answer
																												if cat == "null" and np.random.rand() < 0.6:
																																			continue  # removing 50 % samples
																												writer.write(
																												    {
																												        """input_ids""": ids,
																												        """start_token""": start,
																												        """end_token""": end,
																												        """category""": CATEGORY_MAPPING[cat],
																												    }				)


if __name__ == "__main__":
							from datasets import load_dataset

							from transformers import BigBirdTokenizer

							UpperCAmelCase_							=				load_dataset("""natural_questions""")
							UpperCAmelCase_							=				BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""")

							UpperCAmelCase_							=				data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

							UpperCAmelCase_							=				{
							    '''tokenizer''': tokenizer,
							    '''doc_stride''': DOC_STRIDE,
							    '''max_length''': MAX_LENGTH,
							    '''assertion''': False,
							}
							UpperCAmelCase_							=				data.map(prepare_inputs, fn_kwargs=fn_kwargs)
							UpperCAmelCase_							=				data.remove_columns(["""annotations""", """document""", """id""", """question"""])
							print(data)

							np.random.seed(SEED)
							UpperCAmelCase_							=				'''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
							save_to_disk(data, file_name=cache_file_name)





'''simple docstring'''



def       _UpperCamelCase			(_lowerCamelCase				:  int						)-> int:




     '''simple docstring'''





     __snake_case            =	abs(_lowerCamelCase						)
     __snake_case            =	0
     while n > 0:
          res += n % 10
          n //= 10
     return res







def       _UpperCamelCase			(_lowerCamelCase				:  int						)-> int:




     '''simple docstring'''





     __snake_case            =	abs(_lowerCamelCase						)
     return n if n < 10 else n % 10 + sum_of_digits(n // 10						)







def       _UpperCamelCase			(_lowerCamelCase				:  int						)-> int:




     '''simple docstring'''





     return sum(int(_lowerCamelCase						) for c in str(abs(_lowerCamelCase						)						)						)







def       _UpperCamelCase			()-> None:




     '''simple docstring'''





     from collections.abc import Callable
     from timeit import timeit

     def benchmark_a_function(_lowerCamelCase				:  Callable ,						_lowerCamelCase				:  int						) -> None:
          __snake_case            =	f'''{func.__name__}({value})'''
          __snake_case            =	timeit(f'''__main__.{call}''' ,						setup='''import __main__'''						)
          print(f'''{call:56} = {func(_lowerCamelCase						)} -- {timing:.4f} seconds'''						)

     for value in (26_21_44, 11_25_89_99_06_84_26_24, 1_26_76_50_60_02_28_22_94_01_49_67_03_20_53_76):
          for func in (sum_of_digits, sum_of_digits_recursion, sum_of_digits_compact):
               benchmark_a_function(_lowerCamelCase ,						_lowerCamelCase						)
          print()


if __name__ == "__main__":
     import doctest

     doctest.testmod()
     benchmark()






