
"""simple docstring"""



import os

import jsonlines
import numpy as np
from tqdm import tqdm


UpperCamelCase__          =	20_48
UpperCamelCase__          =	40_96
UpperCamelCase__          =	42
UpperCamelCase__          =	os.environ.pop('PROCESS_TRAIN', 'false')
UpperCamelCase__          =	{'null': 0, 'short': 1, 'long': 2, 'yes': 3, 'no': 4}
def 					lowerCamelCase   (     _snake_case ):
    def choose_first(_snake_case    ,_snake_case=False ):
        assert isinstance(_snake_case    ,_snake_case )
        if len(_snake_case ) == 1:
            UpperCAmelCase__	:      int     							=					answer[0]
            return {k: [answer[k]] for k in answer} if is_long_answer else answer
        for a in answer:
            if is_long_answer:
                UpperCAmelCase__	:      Any     							=					{k: [a[k]] for k in a}
            if len(a['start_token'] ) > 0:
                break
        return a

    UpperCAmelCase__	:      int     							=					{'id': example['id']}
    UpperCAmelCase__	:      List[Any]     							=					example['annotations']
    UpperCAmelCase__	:      Optional[Any]     							=					annotation['yes_no_answer']
    if 0 in yes_no_answer or 1 in yes_no_answer:
        UpperCAmelCase__	:      Any     							=					['yes'] if 1 in yes_no_answer else ['no']
        UpperCAmelCase__	:      str     							=					[]
        UpperCAmelCase__	:      List[Any]     							=					[]
        UpperCAmelCase__	:      str     							=					['<cls>']
    else:
        UpperCAmelCase__	:      List[str]     							=					['short']
        UpperCAmelCase__	:      int     							=					choose_first(annotation['short_answers'] )
        if len(out['start_token'] ) == 0:
            # answer will be long if short is not available
            UpperCAmelCase__	:      List[str]     							=					['long']
            UpperCAmelCase__	:      Any     							=					choose_first(annotation['long_answer']    ,is_long_answer=_snake_case )
            UpperCAmelCase__	:      Optional[int]     							=					[]
        answer.update(_snake_case )

    # disregard some samples
    if len(answer['start_token'] ) > 1 or answer["start_token"] == answer["end_token"]:
        UpperCAmelCase__	:      Optional[int]     							=					True
    else:
        UpperCAmelCase__	:      List[str]     							=					False

    UpperCAmelCase__	:      Dict     							=					['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
    if not all(isinstance(answer[k]    ,_snake_case ) for k in cols ):
        raise ValueError('Issue in ID'    ,example['id'] )

    return answer
def 					lowerCamelCase   (     _snake_case    ,_snake_case=False ):
    UpperCAmelCase__	:      Any     							=					_get_single_answer(_snake_case )
    # bytes are of no use
    del answer["start_byte"]
    del answer["end_byte"]

    # handle yes_no answers explicitly
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        UpperCAmelCase__	:      List[Any]     							=					example['document']['tokens']
        UpperCAmelCase__	:      str     							=					[]
        for i in range(len(doc['token'] ) ):
            if not doc["is_html"][i]:
                context.append(doc['token'][i] )
        return {
            "context": " ".join(_snake_case ),
            "answer": {
                "start_token": -100,  # ignore index in cross-entropy
                "end_token": -100,  # ignore index in cross-entropy
                "category": answer["category"],
                "span": answer["category"],  # extra
            },
        }

    # later, help in removing all no answers
    if answer["start_token"] == [-1]:
        return {
            "context": "None",
            "answer": {
                "start_token": -1,
                "end_token": -1,
                "category": "null",
                "span": "None",  # extra
            },
        }

    # handling normal samples

    UpperCAmelCase__	:      Optional[int]     							=					['start_token', 'end_token']
    answer.update({k: answer[k][0] if len(answer[k] ) > 0 else answer[k] for k in cols} )  # e.g. [10] == 10

    UpperCAmelCase__	:      Any     							=					example['document']['tokens']
    UpperCAmelCase__	:      List[Any]     							=					answer['start_token']
    UpperCAmelCase__	:      List[str]     							=					answer['end_token']

    UpperCAmelCase__	:      Dict     							=					[]
    for i in range(len(doc['token'] ) ):
        if not doc["is_html"][i]:
            context.append(doc['token'][i] )
        else:
            if answer["start_token"] > i:
                start_token -= 1
            if answer["end_token"] > i:
                end_token -= 1
    UpperCAmelCase__	:      List[str]     							=					' '.join(context[start_token:end_token] )

    # checking above code
    if assertion:
        UpperCAmelCase__	:      int     							=					doc['is_html'][answer['start_token'] : answer['end_token']]
        UpperCAmelCase__	:      Optional[Any]     							=					doc['token'][answer['start_token'] : answer['end_token']]
        UpperCAmelCase__	:      Union[str, Any]     							=					' '.join([old[i] for i in range(len(_snake_case ) ) if not is_html[i]] )
        if new != old:
            print('ID:'    ,example['id'] )
            print('New:'    ,_snake_case    ,end='\n' )
            print('Old:'    ,_snake_case    ,end='\n\n' )

    return {
        "context": " ".join(_snake_case ),
        "answer": {
            "start_token": start_token,
            "end_token": end_token - 1,  # this makes it inclusive
            "category": answer["category"],  # either long or short
            "span": new,  # extra
        },
    }
def 					lowerCamelCase   (     _snake_case    ,_snake_case    ,_snake_case=2048    ,_snake_case=4096    ,_snake_case=True ):
    # overlap will be of doc_stride - q_len

    UpperCAmelCase__	:      int     							=					get_context_and_ans(_snake_case    ,assertion=_snake_case )
    UpperCAmelCase__	:      str     							=					out['answer']

    # later, removing these samples
    if answer["start_token"] == -1:
        return {
            "example_id": example["id"],
            "input_ids": [[-1]],
            "labels": {
                "start_token": [-1],
                "end_token": [-1],
                "category": ["null"],
            },
        }

    UpperCAmelCase__	:      Optional[Any]     							=					tokenizer(example['question']['text']    ,out['context'] ).input_ids
    UpperCAmelCase__	:      List[Any]     							=					input_ids.index(tokenizer.sep_token_id ) + 1

    # return yes/no
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        UpperCAmelCase__	:      Tuple     							=					[]
        UpperCAmelCase__	:      Any     							=					[]
        UpperCAmelCase__	:      int     							=					input_ids[:q_len]
        UpperCAmelCase__	:      Tuple     							=					range(_snake_case    ,len(_snake_case )    ,max_length - doc_stride )
        for i in doc_start_indices:
            UpperCAmelCase__	:      str     							=					i + max_length - q_len
            UpperCAmelCase__	:      Optional[Any]     							=					input_ids[i:end_index]
            inputs.append(q_indices + slice )
            category.append(answer['category'][0] )
            if slice[-1] == tokenizer.sep_token_id:
                break

        return {
            "example_id": example["id"],
            "input_ids": inputs,
            "labels": {
                "start_token": [-100] * len(_snake_case ),
                "end_token": [-100] * len(_snake_case ),
                "category": category,
            },
        }

    UpperCAmelCase__	:      Optional[Any]     							=					out['context'].split()
    UpperCAmelCase__	:      List[Any]     							=					splitted_context[answer['end_token']]
    UpperCAmelCase__	:      Dict     							=					len(
        tokenizer(
            ' '.join(splitted_context[: answer['start_token']] )    ,add_special_tokens=_snake_case    ,).input_ids )
    UpperCAmelCase__	:      str     							=					len(
        tokenizer(' '.join(splitted_context[: answer['end_token']] )    ,add_special_tokens=_snake_case ).input_ids )

    answer["start_token"] += q_len
    answer["end_token"] += q_len

    # fixing end token
    UpperCAmelCase__	:      Union[str, Any]     							=					len(tokenizer(_snake_case    ,add_special_tokens=_snake_case ).input_ids )
    if num_sub_tokens > 1:
        answer["end_token"] += num_sub_tokens - 1

    UpperCAmelCase__	:      Union[str, Any]     							=					input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
    UpperCAmelCase__	:      Tuple     							=					answer['start_token']
    UpperCAmelCase__	:      List[Any]     							=					answer['end_token']

    if assertion:
        UpperCAmelCase__	:      Optional[int]     							=					tokenizer.decode(_snake_case )
        if answer["span"] != new:
            print('ISSUE IN TOKENIZATION' )
            print('OLD:'    ,answer['span'] )
            print('NEW:'    ,_snake_case    ,end='\n\n' )

    if len(_snake_case ) <= max_length:
        return {
            "example_id": example["id"],
            "input_ids": [input_ids],
            "labels": {
                "start_token": [answer["start_token"]],
                "end_token": [answer["end_token"]],
                "category": answer["category"],
            },
        }

    UpperCAmelCase__	:      List[str]     							=					input_ids[:q_len]
    UpperCAmelCase__	:      Any     							=					range(_snake_case    ,len(_snake_case )    ,max_length - doc_stride )

    UpperCAmelCase__	:      Optional[Any]     							=					[]
    UpperCAmelCase__	:      Any     							=					[]
    UpperCAmelCase__	:      Union[str, Any]     							=					[]
    UpperCAmelCase__	:      Optional[int]     							=					[]  # null, yes, no, long, short
    for i in doc_start_indices:
        UpperCAmelCase__	:      Union[str, Any]     							=					i + max_length - q_len
        UpperCAmelCase__	:      Dict     							=					input_ids[i:end_index]
        inputs.append(q_indices + slice )
        assert len(inputs[-1] ) <= max_length, "Issue in truncating length"

        if start_token >= i and end_token <= end_index - 1:
            UpperCAmelCase__	:      Union[str, Any]     							=					start_token - i + q_len
            UpperCAmelCase__	:      Optional[int]     							=					end_token - i + q_len
            answers_category.append(answer['category'][0] )  # ["short"] -> "short"
        else:
            UpperCAmelCase__	:      Union[str, Any]     							=					-100
            UpperCAmelCase__	:      int     							=					-100
            answers_category.append('null' )
        UpperCAmelCase__	:      Any     							=					inputs[-1][start_token : end_token + 1]

        answers_start_token.append(_snake_case )
        answers_end_token.append(_snake_case )
        if assertion:
            if new != old and new != [tokenizer.cls_token_id]:
                print('ISSUE in strided for ID:'    ,example['id'] )
                print('New:'    ,tokenizer.decode(_snake_case ) )
                print('Old:'    ,tokenizer.decode(_snake_case )    ,end='\n\n' )
        if slice[-1] == tokenizer.sep_token_id:
            break

    return {
        "example_id": example["id"],
        "input_ids": inputs,
        "labels": {
            "start_token": answers_start_token,
            "end_token": answers_end_token,
            "category": answers_category,
        },
    }
def 					lowerCamelCase   (     _snake_case    ,_snake_case    ,_snake_case=2048    ,_snake_case=4096    ,_snake_case=False ):
    UpperCAmelCase__	:      Union[str, Any]     							=					get_strided_contexts_and_ans(
        _snake_case    ,_snake_case    ,doc_stride=_snake_case    ,max_length=_snake_case    ,assertion=_snake_case    ,)

    return example
def 					lowerCamelCase   (     _snake_case    ,_snake_case ):
    with jsonlines.open(_snake_case    ,'a' ) as writer:
        for example in tqdm(_snake_case    ,total=len(_snake_case )    ,desc='Saving samples ... ' ):
            UpperCAmelCase__	:      Optional[Any]     							=					example['labels']
            for ids, start, end, cat in zip(
                example['input_ids']    ,labels['start_token']    ,labels['end_token']    ,labels['category']    ,):
                if start == -1 and end == -1:
                    continue  # leave waste samples with no answer
                if cat == "null" and np.random.rand() < 0.6:
                    continue  # removing 50 % samples
                writer.write(
                    {
                        'input_ids': ids,
                        'start_token': start,
                        'end_token': end,
                        'category': CATEGORY_MAPPING[cat],
                    } )


if __name__ == "__main__":
    from datasets import load_dataset

    from transformers import BigBirdTokenizer

    UpperCamelCase__          =	load_dataset('natural_questions')
    UpperCamelCase__          =	BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base')

    UpperCamelCase__          =	data['train' if PROCESS_TRAIN == 'true' else 'validation']

    UpperCamelCase__          =	{
        'tokenizer': tokenizer,
        'doc_stride': DOC_STRIDE,
        'max_length': MAX_LENGTH,
        'assertion': False,
    }
    UpperCamelCase__          =	data.map(prepare_inputs, fn_kwargs=fn_kwargs)
    UpperCamelCase__          =	data.remove_columns(['annotations', 'document', 'id', 'question'])
    print(data)

    np.random.seed(SEED)
    UpperCamelCase__          =	'nq-training.jsonl' if PROCESS_TRAIN == 'true' else 'nq-validation.jsonl'
    save_to_disk(data, file_name=cache_file_name)



"""simple docstring"""



from typing import Any





class 				a			:



      def __init__(   self     ,	UpperCamelCase_      ):
          UpperCAmelCase__	:      Optional[Any]     							=					data
          UpperCAmelCase__	:      List[str]     							=					None



      def __repr__(   self      ):
          return F'''Node({self.data})'''





class 				a			:



      def __init__(   self      ):
          UpperCAmelCase__	:      Any     							=					None



      def __iter__(   self      ):
          UpperCAmelCase__	:      List[str]     							=					self.head
          while node:
              yield node.data
              UpperCAmelCase__	:      Optional[int]     							=					node.next



      def __len__(   self      ):
          return sum(1 for _ in self      )



      def __repr__(   self      ):
          return "->".join([str(UpperCamelCase_      ) for item in self]      )



      def __getitem__(   self     ,	UpperCamelCase_      ):
          if not 0 <= index < len(self      ):
              raise ValueError('list index out of range.'      )
          for i, node in enumerate(self      ):
              if i == index:
                  return node
          return None



      def __setitem__(   self     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          if not 0 <= index < len(self      ):
              raise ValueError('list index out of range.'      )
          UpperCAmelCase__	:      List[str]     							=					self.head
          for _ in range(UpperCamelCase_      ):
              UpperCAmelCase__	:      Optional[Any]     							=					current.next
          UpperCAmelCase__	:      List[str]     							=					data



      def  __snake_case							(   self     ,	UpperCamelCase_      ):
          self.insert_nth(len(self      )     ,	UpperCamelCase_      )



      def  __snake_case							(   self     ,	UpperCamelCase_      ):
          self.insert_nth(0     ,	UpperCamelCase_      )



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          if not 0 <= index <= len(self      ):
              raise IndexError('list index out of range'      )
          UpperCAmelCase__	:      str     							=					Node(UpperCamelCase_      )
          if self.head is None:
              UpperCAmelCase__	:      Tuple     							=					new_node
          elif index == 0:
              UpperCAmelCase__	:      Optional[int]     							=					self.head  # link new_node to head
              UpperCAmelCase__	:      Any     							=					new_node
          else:
              UpperCAmelCase__	:      Dict     							=					self.head
              for _ in range(index - 1      ):
                  UpperCAmelCase__	:      Tuple     							=					temp.next
              UpperCAmelCase__	:      int     							=					temp.next
              UpperCAmelCase__	:      Tuple     							=					new_node



      def  __snake_case							(   self      ):  # print every node data
          print(self      )



      def  __snake_case							(   self      ):
          return self.delete_nth(0      )



      def  __snake_case							(   self      ):  # delete from tail
          return self.delete_nth(len(self      ) - 1      )



      def  __snake_case							(   self     ,	UpperCamelCase_ = 0      ):
          if not 0 <= index <= len(self      ) - 1:  # test if index is valid
              raise IndexError('List index out of range.'      )
          UpperCAmelCase__	:      Union[str, Any]     							=					self.head  # default first node
          if index == 0:
              UpperCAmelCase__	:      Dict     							=					self.head.next
          else:
              UpperCAmelCase__	:      List[Any]     							=					self.head
              for _ in range(index - 1      ):
                  UpperCAmelCase__	:      Any     							=					temp.next
              UpperCAmelCase__	:      Dict     							=					temp.next
              UpperCAmelCase__	:      Tuple     							=					temp.next.next
          return delete_node.data



      def  __snake_case							(   self      ):
          return self.head is None



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Optional[Any]     							=					None
          UpperCAmelCase__	:      str     							=					self.head

          while current:
              # Store the current node's next node.
              UpperCAmelCase__	:      Any     							=					current.next
              # Make the current node's next point backwards
              UpperCAmelCase__	:      Optional[int]     							=					prev
              # Make the previous node be the current node
              UpperCAmelCase__	:      List[Any]     							=					current
              # Make the current node the next node (to progress iteration)
              UpperCAmelCase__	:      int     							=					next_node
          # Return prev in order to put the head at the end
          UpperCAmelCase__	:      str     							=					prev




def 					lowerCamelCase   (     ):
    UpperCAmelCase__	:      Dict     							=					LinkedList()
    assert linked_list.is_empty() is True
    assert str(_snake_case ) == ""

    try:
        linked_list.delete_head()
        raise AssertionError  # This should not happen.
    except IndexError:
        assert True  # This should happen.

    try:
        linked_list.delete_tail()
        raise AssertionError  # This should not happen.
    except IndexError:
        assert True  # This should happen.

    for i in range(10 ):
        assert len(_snake_case ) == i
        linked_list.insert_nth(_snake_case    ,i + 1 )
    assert str(_snake_case ) == "->".join(str(_snake_case ) for i in range(1    ,11 ) )

    linked_list.insert_head(0 )
    linked_list.insert_tail(11 )
    assert str(_snake_case ) == "->".join(str(_snake_case ) for i in range(0    ,12 ) )

    assert linked_list.delete_head() == 0
    assert linked_list.delete_nth(9 ) == 10
    assert linked_list.delete_tail() == 11
    assert len(_snake_case ) == 9
    assert str(_snake_case ) == "->".join(str(_snake_case ) for i in range(1    ,10 ) )

    assert all(linked_list[i] == i + 1 for i in range(0    ,9 ) ) is True

    for i in range(0    ,9 ):
        UpperCAmelCase__	:      Any     							=					-i
    assert all(linked_list[i] == -i for i in range(0    ,9 ) ) is True

    linked_list.reverse()
    assert str(_snake_case ) == "->".join(str(_snake_case ) for i in range(-8    ,1 ) )
def 					lowerCamelCase   (     ):
    UpperCAmelCase__	:      int     							=					[
        -9,
        100,
        Node(77345112 ),
        'dlrow olleH',
        7,
        5555,
        0,
        -192.55555,
        'Hello, world!',
        77.9,
        Node(10 ),
        None,
        None,
        12.20,
    ]
    UpperCAmelCase__	:      Dict     							=					LinkedList()

    for i in test_input:
        linked_list.insert_tail(_snake_case )

    # Check if it's empty or not
    assert linked_list.is_empty() is False
    assert (
        str(_snake_case ) == "-9->100->Node(77345112)->dlrow olleH->7->5555->0->"
        "-192.55555->Hello, world!->77.9->Node(10)->None->None->12.2"
    )

    # Delete the head
    UpperCAmelCase__	:      Any     							=					linked_list.delete_head()
    assert result == -9
    assert (
        str(_snake_case ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
        "Hello, world!->77.9->Node(10)->None->None->12.2"
    )

    # Delete the tail
    UpperCAmelCase__	:      Union[str, Any]     							=					linked_list.delete_tail()
    assert result == 12.2
    assert (
        str(_snake_case ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
        "Hello, world!->77.9->Node(10)->None->None"
    )

    # Delete a node in specific location in linked list
    UpperCAmelCase__	:      Dict     							=					linked_list.delete_nth(10 )
    assert result is None
    assert (
        str(_snake_case ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
        "Hello, world!->77.9->Node(10)->None"
    )

    # Add a Node instance to its head
    linked_list.insert_head(Node('Hello again, world!' ) )
    assert (
        str(_snake_case )
        == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
        "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None"
    )

    # Add None to its tail
    linked_list.insert_tail(_snake_case )
    assert (
        str(_snake_case )
        == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
        "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None->None"
    )

    # Reverse the linked list
    linked_list.reverse()
    assert (
        str(_snake_case )
        == "None->None->Node(10)->77.9->Hello, world!->-192.55555->0->5555->"
        "7->dlrow olleH->Node(77345112)->100->Node(Hello again, world!)"
    )
def 					lowerCamelCase   (     ):
    from doctest import testmod

    testmod()

    UpperCAmelCase__	:      Optional[Any]     							=					LinkedList()
    linked_list.insert_head(input('Inserting 1st at head ' ).strip() )
    linked_list.insert_head(input('Inserting 2nd at head ' ).strip() )
    print('\nPrint list:' )
    linked_list.print_list()
    linked_list.insert_tail(input('\nInserting 1st at tail ' ).strip() )
    linked_list.insert_tail(input('Inserting 2nd at tail ' ).strip() )
    print('\nPrint list:' )
    linked_list.print_list()
    print('\nDelete head' )
    linked_list.delete_head()
    print('Delete tail' )
    linked_list.delete_tail()
    print('\nPrint list:' )
    linked_list.print_list()
    print('\nReverse linked list' )
    linked_list.reverse()
    print('\nPrint list:' )
    linked_list.print_list()
    print('\nString representation of linked list:' )
    print(_snake_case )
    print('\nReading/changing Node data using indexing:' )
    print(F'''Element at Position 1: {linked_list[1]}''' )
    UpperCAmelCase__	:      List[Any]     							=					input('Enter New Value: ' ).strip()
    print('New list:' )
    print(_snake_case )
    print(F'''length of linked_list is : {len(_snake_case )}''' )


if __name__ == "__main__":
    main()

