


'''simple docstring'''






import os

import jsonlines
import numpy as np
from tqdm import tqdm


a_					:    List[Any]          =    2_0_4_8
a_					:    List[Any]          =    4_0_9_6
a_					:    int          =    4_2
a_					:    Tuple          =    os.environ.pop("PROCESS_TRAIN", "false")
a_					:    Optional[int]          =    {"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}






def 	_A				(lowerCAmelCase__  :List[str]		)   ->  str:




     '''simple docstring'''



     def choose_first(lowerCAmelCase__  :Optional[int]				,						lowerCAmelCase__  :List[Any]=False		):
          assert isinstance(lowerCAmelCase__				,						lowerCAmelCase__		)
          if len(lowerCAmelCase__		) == 1:
               _a			      =							answer[0]
               return {k: [answer[k]] for k in answer} if is_long_answer else answer
          for a in answer:
               if is_long_answer:
                    _a			      =							{k: [a[k]] for k in a}
               if len(a['start_token']		) > 0:
                    break
          return a

     _a			      =							{'id': example['id']}
     _a			      =							example['annotations']
     _a			      =							annotation['yes_no_answer']
     if 0 in yes_no_answer or 1 in yes_no_answer:
          _a			      =							['yes'] if 1 in yes_no_answer else ['no']
          _a			      =							_a			      =							[]
          _a			      =							_a			      =							[]
          _a			      =							['<cls>']
     else:
          _a			      =							['short']
          _a			      =							choose_first(annotation['short_answers']		)
          if len(out['start_token']		) == 0:
               # answer will be long if short is not available
               _a			      =							['long']
               _a			      =							choose_first(annotation['long_answer']				,						is_long_answer=lowerCAmelCase__		)
               _a			      =							[]
          answer.update(lowerCAmelCase__		)

     # disregard some samples
     if len(answer['start_token']		) > 1 or answer["start_token"] == answer["end_token"]:
          _a			      =							True
     else:
          _a			      =							False

     _a			      =							['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
     if not all(isinstance(answer[k]				,						lowerCAmelCase__		) for k in cols		):
          raise ValueError('Issue in ID'				,						example['id']		)

     return answer






def 	_A				(lowerCAmelCase__  :Union[str, Any]				,						lowerCAmelCase__  :Tuple=False		)   ->  Optional[int]:




     '''simple docstring'''



     _a			      =							_get_single_answer(lowerCAmelCase__		)
     # bytes are of no use
     del answer["start_byte"]
     del answer["end_byte"]

     # handle yes_no answers explicitly
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          _a			      =							example['document']['tokens']
          _a			      =							[]
          for i in range(len(doc['token']		)		):
               if not doc["is_html"][i]:
                    context.append(doc['token'][i]		)
          return {
              "context": " ".join(lowerCAmelCase__		),
              "answer": {
                  "start_token": -1_00,  # ignore index in cross-entropy
                  "end_token": -1_00,  # ignore index in cross-entropy
                  "category": answer["category"],
                  "span": answer["category"],  # extra
              },
          }

     # later, help in removing all no answers
     if answer["start_token"] == [-1]:
          return {
              "context": "None",
              "answer": {
                  "start_token": -1,
                  "end_token": -1,
                  "category": "null",
                  "span": "None",  # extra
              },
          }

     # handling normal samples

     _a			      =							['start_token', 'end_token']
     answer.update({k: answer[k][0] if len(answer[k]		) > 0 else answer[k] for k in cols}		)  # e.g. [10] == 10

     _a			      =							example['document']['tokens']
     _a			      =							answer['start_token']
     _a			      =							answer['end_token']

     _a			      =							[]
     for i in range(len(doc['token']		)		):
          if not doc["is_html"][i]:
               context.append(doc['token'][i]		)
          else:
               if answer["start_token"] > i:
                    start_token -= 1
               if answer["end_token"] > i:
                    end_token -= 1
     _a			      =							' '.join(context[start_token:end_token]		)

     # checking above code
     if assertion:
          _a			      =							doc['is_html'][answer['start_token'] : answer['end_token']]
          _a			      =							doc['token'][answer['start_token'] : answer['end_token']]
          _a			      =							' '.join([old[i] for i in range(len(lowerCAmelCase__		)		) if not is_html[i]]		)
          if new != old:
               print('ID:'				,						example['id']		)
               print('New:'				,						lowerCAmelCase__				,						end='\n'		)
               print('Old:'				,						lowerCAmelCase__				,						end='\n\n'		)

     return {
         "context": " ".join(lowerCAmelCase__		),
         "answer": {
             "start_token": start_token,
             "end_token": end_token - 1,  # this makes it inclusive
             "category": answer["category"],  # either long or short
             "span": new,  # extra
         },
     }






def 	_A				(lowerCAmelCase__  :Dict				,						lowerCAmelCase__  :Dict				,						lowerCAmelCase__  :str=20_48				,						lowerCAmelCase__  :int=40_96				,						lowerCAmelCase__  :str=True		)   ->  str:




     '''simple docstring'''



     _a			      =							get_context_and_ans(lowerCAmelCase__				,						assertion=lowerCAmelCase__		)
     _a			      =							out['answer']

     # later, removing these samples
     if answer["start_token"] == -1:
          return {
              "example_id": example["id"],
              "input_ids": [[-1]],
              "labels": {
                  "start_token": [-1],
                  "end_token": [-1],
                  "category": ["null"],
              },
          }

     _a			      =							tokenizer(example['question']['text']				,						out['context']		).input_ids
     _a			      =							input_ids.index(tokenizer.sep_token_id		) + 1

     # return yes/no
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          _a			      =							[]
          _a			      =							[]
          _a			      =							input_ids[:q_len]
          _a			      =							range(lowerCAmelCase__				,						len(lowerCAmelCase__		)				,						max_length - doc_stride		)
          for i in doc_start_indices:
               _a			      =							i + max_length - q_len
               _a			      =							input_ids[i:end_index]
               inputs.append(q_indices + slice		)
               category.append(answer['category'][0]		)
               if slice[-1] == tokenizer.sep_token_id:
                    break

          return {
              "example_id": example["id"],
              "input_ids": inputs,
              "labels": {
                  "start_token": [-1_00] * len(lowerCAmelCase__		),
                  "end_token": [-1_00] * len(lowerCAmelCase__		),
                  "category": category,
              },
          }

     _a			      =							out['context'].split()
     _a			      =							splitted_context[answer['end_token']]
     _a			      =							len(
         tokenizer(
             ' '.join(splitted_context[: answer['start_token']]		)				,						add_special_tokens=lowerCAmelCase__				,						).input_ids		)
     _a			      =							len(
         tokenizer(' '.join(splitted_context[: answer['end_token']]		)				,						add_special_tokens=lowerCAmelCase__		).input_ids		)

     answer["start_token"] += q_len
     answer["end_token"] += q_len

     # fixing end token
     _a			      =							len(tokenizer(lowerCAmelCase__				,						add_special_tokens=lowerCAmelCase__		).input_ids		)
     if num_sub_tokens > 1:
          answer["end_token"] += num_sub_tokens - 1

     _a			      =							input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
     _a			      =							answer['start_token']
     _a			      =							answer['end_token']

     if assertion:
          _a			      =							tokenizer.decode(lowerCAmelCase__		)
          if answer["span"] != new:
               print('ISSUE IN TOKENIZATION'		)
               print('OLD:'				,						answer['span']		)
               print('NEW:'				,						lowerCAmelCase__				,						end='\n\n'		)

     if len(lowerCAmelCase__		) <= max_length:
          return {
              "example_id": example["id"],
              "input_ids": [input_ids],
              "labels": {
                  "start_token": [answer["start_token"]],
                  "end_token": [answer["end_token"]],
                  "category": answer["category"],
              },
          }

     _a			      =							input_ids[:q_len]
     _a			      =							range(lowerCAmelCase__				,						len(lowerCAmelCase__		)				,						max_length - doc_stride		)

     _a			      =							[]
     _a			      =							[]
     _a			      =							[]
     _a			      =							[]  # null, yes, no, long, short
     for i in doc_start_indices:
          _a			      =							i + max_length - q_len
          _a			      =							input_ids[i:end_index]
          inputs.append(q_indices + slice		)
          assert len(inputs[-1]		) <= max_length, "Issue in truncating length"

          if start_token >= i and end_token <= end_index - 1:
               _a			      =							start_token - i + q_len
               _a			      =							end_token - i + q_len
               answers_category.append(answer['category'][0]		)  # ["short"] -> "short"
          else:
               _a			      =							-1_00
               _a			      =							-1_00
               answers_category.append('null'		)
          _a			      =							inputs[-1][start_token : end_token + 1]

          answers_start_token.append(lowerCAmelCase__		)
          answers_end_token.append(lowerCAmelCase__		)
          if assertion:
               if new != old and new != [tokenizer.cls_token_id]:
                    print('ISSUE in strided for ID:'				,						example['id']		)
                    print('New:'				,						tokenizer.decode(lowerCAmelCase__		)		)
                    print('Old:'				,						tokenizer.decode(lowerCAmelCase__		)				,						end='\n\n'		)
          if slice[-1] == tokenizer.sep_token_id:
               break

     return {
         "example_id": example["id"],
         "input_ids": inputs,
         "labels": {
             "start_token": answers_start_token,
             "end_token": answers_end_token,
             "category": answers_category,
         },
     }






def 	_A				(lowerCAmelCase__  :Dict				,						lowerCAmelCase__  :Optional[Any]				,						lowerCAmelCase__  :Any=20_48				,						lowerCAmelCase__  :Tuple=40_96				,						lowerCAmelCase__  :Dict=False		)   ->  str:




     '''simple docstring'''



     _a			      =							get_strided_contexts_and_ans(
         lowerCAmelCase__				,						lowerCAmelCase__				,						doc_stride=lowerCAmelCase__				,						max_length=lowerCAmelCase__				,						assertion=lowerCAmelCase__				,						)

     return example






def 	_A				(lowerCAmelCase__  :int				,						lowerCAmelCase__  :List[str]		)   ->  Any:




     '''simple docstring'''



     with jsonlines.open(lowerCAmelCase__				,						'a'		) as writer:
          for example in tqdm(lowerCAmelCase__				,						total=len(lowerCAmelCase__		)				,						desc='Saving samples ... '		):
               _a			      =							example['labels']
               for ids, start, end, cat in zip(
                   example['input_ids']				,						labels['start_token']				,						labels['end_token']				,						labels['category']				,						):
                    if start == -1 and end == -1:
                         continue  # leave waste samples with no answer
                    if cat == "null" and np.random.rand() < 0.6:
                         continue  # removing 50 % samples
                    writer.write(
                        {
                            'input_ids': ids,
                            'start_token': start,
                            'end_token': end,
                            'category': CATEGORY_MAPPING[cat],
                        }		)


if __name__ == "__main__":
       from datasets import load_dataset

       from transformers import BigBirdTokenizer

       a_					:    List[str]          =    load_dataset("natural_questions")
       a_					:    List[str]          =    BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

       a_					:    str          =    data["train" if PROCESS_TRAIN == "true" else "validation"]

       a_					:    Union[str, Any]          =    {
           "tokenizer": tokenizer,
           "doc_stride": DOC_STRIDE,
           "max_length": MAX_LENGTH,
           "assertion": False,
       }
       a_					:    Any          =    data.map(prepare_inputs, fn_kwargs=fn_kwargs)
       a_					:    Dict          =    data.remove_columns(["annotations", "document", "id", "question"])
       print(data)

       np.random.seed(SEED)
       a_					:    str          =    "nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
       save_to_disk(data, file_name=cache_file_name)





'''simple docstring'''






import pytest

from datasets.splits import SplitDict, SplitInfo
from datasets.utils.py_utils import asdict






@pytest.mark.parametrize(
    'split_dict'				,						[
        SplitDict(),
        SplitDict({'train': SplitInfo(name='train'				,						num_bytes=13_37				,						num_examples=42				,						dataset_name='my_dataset'		)}		),
        SplitDict({'train': SplitInfo(name='train'				,						num_bytes=13_37				,						num_examples=42		)}		),
        SplitDict({'train': SplitInfo()}		),
    ]				,						)
def 	_A				(lowerCAmelCase__  :SplitDict		)   ->  Any:




     '''simple docstring'''



     _a			      =							split_dict._to_yaml_list()
     assert len(lowerCAmelCase__		) == len(lowerCAmelCase__		)
     _a			      =							SplitDict._from_yaml_list(lowerCAmelCase__		)
     for split_name, split_info in split_dict.items():
          # dataset_name field is deprecated, and is therefore not part of the YAML dump
          _a			      =							None
          # the split name of split_dict takes over the name of the split info object
          _a			      =							split_name
     assert split_dict == reloaded






@pytest.mark.parametrize(
    'split_info'				,						[SplitInfo(), SplitInfo(dataset_name=lowerCAmelCase__		), SplitInfo(dataset_name='my_dataset'		)]		)
def 	_A				(lowerCAmelCase__  :Optional[Any]		)   ->  List[str]:




     '''simple docstring'''



     _a			      =							asdict(SplitDict({'train': split_info}		)		)
     assert "dataset_name" in split_dict_asdict["train"]
     assert split_dict_asdict["train"]["dataset_name"] == split_info.dataset_name

