



import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowercase_	 =						2_048
lowercase_	 =						4_096
lowercase_	 =						42
lowercase_	 =						os.environ.pop("PROCESS_TRAIN", "false")
lowercase_	 =						{"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}




def lowerCAmelCase						(__A):



     """simple docstring"""



     def choose_first(__A						,			__A=False):
          assert isinstance(_UpperCamelCase						,			_UpperCamelCase)
          if len(_UpperCamelCase) == 1:
               _a											=						answer[0]
               return {k: [answer[k]] for k in answer} if is_long_answer else answer
          for a in answer:
               if is_long_answer:
                    _a											=						{k: [a[k]] for k in a}
               if len(a['''start_token''']) > 0:
                    break
          return a

     _a											=						{'''id''': example['''id''']}
     _a											=						example['''annotations''']
     _a											=						annotation['''yes_no_answer''']
     if 0 in yes_no_answer or 1 in yes_no_answer:
          _a											=						['''yes'''] if 1 in yes_no_answer else ['''no''']
          _a											=						_a											=						[]
          _a											=						_a											=						[]
          _a											=						['''<cls>''']
     else:
          _a											=						['''short''']
          _a											=						choose_first(annotation['''short_answers'''])
          if len(out['''start_token''']) == 0:
               # answer will be long if short is not available
               _a											=						['''long''']
               _a											=						choose_first(annotation['''long_answer''']						,			is_long_answer=_UpperCamelCase)
               _a											=						[]
          answer.update(_UpperCamelCase)

     # disregard some samples
     if len(answer['''start_token''']) > 1 or answer["start_token"] == answer["end_token"]:
          _a											=						True
     else:
          _a											=						False

     _a											=						['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
     if not all(isinstance(answer[k]						,			_UpperCamelCase) for k in cols):
          raise ValueError('''Issue in ID'''						,			example['''id'''])

     return answer




def lowerCAmelCase						(__A						,			__A=False):



     """simple docstring"""



     _a											=						_get_single_answer(_UpperCamelCase)
     # bytes are of no use
     del answer["start_byte"]
     del answer["end_byte"]

     # handle yes_no answers explicitly
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          _a											=						example['''document''']['''tokens''']
          _a											=						[]
          for i in range(len(doc['''token'''])):
               if not doc["is_html"][i]:
                    context.append(doc['''token'''][i])
          return {
              "context": " ".join(_UpperCamelCase),
              "answer": {
                  "start_token": -100,  # ignore index in cross-entropy
                  "end_token": -100,  # ignore index in cross-entropy
                  "category": answer["category"],
                  "span": answer["category"],  # extra
              },
          }

     # later, help in removing all no answers
     if answer["start_token"] == [-1]:
          return {
              "context": "None",
              "answer": {
                  "start_token": -1,
                  "end_token": -1,
                  "category": "null",
                  "span": "None",  # extra
              },
          }

     # handling normal samples

     _a											=						['''start_token''', '''end_token''']
     answer.update({k: answer[k][0] if len(answer[k]) > 0 else answer[k] for k in cols})  # e.g. [10] == 10

     _a											=						example['''document''']['''tokens''']
     _a											=						answer['''start_token''']
     _a											=						answer['''end_token''']

     _a											=						[]
     for i in range(len(doc['''token'''])):
          if not doc["is_html"][i]:
               context.append(doc['''token'''][i])
          else:
               if answer["start_token"] > i:
                    start_token -= 1
               if answer["end_token"] > i:
                    end_token -= 1
     _a											=						''' '''.join(context[start_token:end_token])

     # checking above code
     if assertion:
          _a											=						doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
          _a											=						doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
          _a											=						''' '''.join([old[i] for i in range(len(_UpperCamelCase)) if not is_html[i]])
          if new != old:
               print('''ID:'''						,			example['''id'''])
               print('''New:'''						,			_UpperCamelCase						,			end='''\n''')
               print('''Old:'''						,			_UpperCamelCase						,			end='''\n\n''')

     return {
         "context": " ".join(_UpperCamelCase),
         "answer": {
             "start_token": start_token,
             "end_token": end_token - 1,  # this makes it inclusive
             "category": answer["category"],  # either long or short
             "span": new,  # extra
         },
     }




def lowerCAmelCase						(__A						,			__A						,			__A=2_048						,			__A=4_096						,			__A=True):



     """simple docstring"""



     _a											=						get_context_and_ans(_UpperCamelCase						,			assertion=_UpperCamelCase)
     _a											=						out['''answer''']

     # later, removing these samples
     if answer["start_token"] == -1:
          return {
              "example_id": example["id"],
              "input_ids": [[-1]],
              "labels": {
                  "start_token": [-1],
                  "end_token": [-1],
                  "category": ["null"],
              },
          }

     _a											=						tokenizer(example['''question''']['''text''']						,			out['''context''']).input_ids
     _a											=						input_ids.index(tokenizer.sep_token_id) + 1

     # return yes/no
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          _a											=						[]
          _a											=						[]
          _a											=						input_ids[:q_len]
          _a											=						range(_UpperCamelCase						,			len(_UpperCamelCase)						,			max_length - doc_stride)
          for i in doc_start_indices:
               _a											=						i + max_length - q_len
               _a											=						input_ids[i:end_index]
               inputs.append(q_indices + slice)
               category.append(answer['''category'''][0])
               if slice[-1] == tokenizer.sep_token_id:
                    break

          return {
              "example_id": example["id"],
              "input_ids": inputs,
              "labels": {
                  "start_token": [-100] * len(_UpperCamelCase),
                  "end_token": [-100] * len(_UpperCamelCase),
                  "category": category,
              },
          }

     _a											=						out['''context'''].split()
     _a											=						splitted_context[answer['''end_token''']]
     _a											=						len(
         tokenizer(
             ''' '''.join(splitted_context[: answer['''start_token''']])						,			add_special_tokens=_UpperCamelCase						,			).input_ids)
     _a											=						len(
         tokenizer(''' '''.join(splitted_context[: answer['''end_token''']])						,			add_special_tokens=_UpperCamelCase).input_ids)

     answer["start_token"] += q_len
     answer["end_token"] += q_len

     # fixing end token
     _a											=						len(tokenizer(_UpperCamelCase						,			add_special_tokens=_UpperCamelCase).input_ids)
     if num_sub_tokens > 1:
          answer["end_token"] += num_sub_tokens - 1

     _a											=						input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
     _a											=						answer['''start_token''']
     _a											=						answer['''end_token''']

     if assertion:
          _a											=						tokenizer.decode(_UpperCamelCase)
          if answer["span"] != new:
               print('''ISSUE IN TOKENIZATION''')
               print('''OLD:'''						,			answer['''span'''])
               print('''NEW:'''						,			_UpperCamelCase						,			end='''\n\n''')

     if len(_UpperCamelCase) <= max_length:
          return {
              "example_id": example["id"],
              "input_ids": [input_ids],
              "labels": {
                  "start_token": [answer["start_token"]],
                  "end_token": [answer["end_token"]],
                  "category": answer["category"],
              },
          }

     _a											=						input_ids[:q_len]
     _a											=						range(_UpperCamelCase						,			len(_UpperCamelCase)						,			max_length - doc_stride)

     _a											=						[]
     _a											=						[]
     _a											=						[]
     _a											=						[]  # null, yes, no, long, short
     for i in doc_start_indices:
          _a											=						i + max_length - q_len
          _a											=						input_ids[i:end_index]
          inputs.append(q_indices + slice)
          assert len(inputs[-1]) <= max_length, "Issue in truncating length"

          if start_token >= i and end_token <= end_index - 1:
               _a											=						start_token - i + q_len
               _a											=						end_token - i + q_len
               answers_category.append(answer['''category'''][0])  # ["short"] -> "short"
          else:
               _a											=						-100
               _a											=						-100
               answers_category.append('''null''')
          _a											=						inputs[-1][start_token : end_token + 1]

          answers_start_token.append(_UpperCamelCase)
          answers_end_token.append(_UpperCamelCase)
          if assertion:
               if new != old and new != [tokenizer.cls_token_id]:
                    print('''ISSUE in strided for ID:'''						,			example['''id'''])
                    print('''New:'''						,			tokenizer.decode(_UpperCamelCase))
                    print('''Old:'''						,			tokenizer.decode(_UpperCamelCase)						,			end='''\n\n''')
          if slice[-1] == tokenizer.sep_token_id:
               break

     return {
         "example_id": example["id"],
         "input_ids": inputs,
         "labels": {
             "start_token": answers_start_token,
             "end_token": answers_end_token,
             "category": answers_category,
         },
     }




def lowerCAmelCase						(__A						,			__A						,			__A=2_048						,			__A=4_096						,			__A=False):



     """simple docstring"""



     _a											=						get_strided_contexts_and_ans(
         _UpperCamelCase						,			_UpperCamelCase						,			doc_stride=_UpperCamelCase						,			max_length=_UpperCamelCase						,			assertion=_UpperCamelCase						,			)

     return example




def lowerCAmelCase						(__A						,			__A):



     """simple docstring"""



     with jsonlines.open(_UpperCamelCase						,			'''a''') as writer:
          for example in tqdm(_UpperCamelCase						,			total=len(_UpperCamelCase)						,			desc='''Saving samples ... '''):
               _a											=						example['''labels''']
               for ids, start, end, cat in zip(
                   example['''input_ids''']						,			labels['''start_token''']						,			labels['''end_token''']						,			labels['''category''']						,			):
                    if start == -1 and end == -1:
                         continue  # leave waste samples with no answer
                    if cat == "null" and np.random.rand() < 0.6:
                         continue  # removing 50 % samples
                    writer.write(
                        {
                            '''input_ids''': ids,
                            '''start_token''': start,
                            '''end_token''': end,
                            '''category''': CATEGORY_MAPPING[cat],
                        })


if __name__ == "__main__":
  from datasets import load_dataset

  from transformers import BigBirdTokenizer

  lowercase_	 =						load_dataset("natural_questions")
  lowercase_	 =						BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

  lowercase_	 =						data["train" if PROCESS_TRAIN == "true" else "validation"]

  lowercase_	 =						{
      "tokenizer": tokenizer,
      "doc_stride": DOC_STRIDE,
      "max_length": MAX_LENGTH,
      "assertion": False,
  }
  lowercase_	 =						data.map(prepare_inputs, fn_kwargs=fn_kwargs)
  lowercase_	 =						data.remove_columns(["annotations", "document", "id", "question"])
  print(data)

  np.random.seed(SEED)
  lowercase_	 =						"nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
  save_to_disk(data, file_name=cache_file_name)








'''simple docstring'''





from __future__ import annotations

import typing
from collections.abc import Iterable

import numpy as np

lowercase_	 =						typing.Union[Iterable[float], Iterable[int], np.ndarray]  # noqa: UP007
lowercase_	 =						typing.Union[np.floataa, int, float]  # noqa: UP007




def lowerCAmelCase						(__A						,			__A):



     """simple docstring"""



     return np.sqrt(np.sum((np.asarray(__A) - np.asarray(__A)) ** 2))




def lowerCAmelCase						(__A						,			__A):



     """simple docstring"""



     return sum((va - va) ** 2 for va, va in zip(__A						,			__A)) ** (1 / 2)


if __name__ == "__main__":




  def lowerCAmelCase						():



       """simple docstring"""



       from timeit import timeit

       print('''Without Numpy''')
       print(
           timeit(
               '''euclidean_distance_no_np([1, 2, 3], [4, 5, 6])'''						,			number=10_000						,			globals=globals()						,			))
       print('''With Numpy''')
       print(
           timeit(
               '''euclidean_distance([1, 2, 3], [4, 5, 6])'''						,			number=10_000						,			globals=globals()						,			))
  benchmark()



