
'''simple docstring'''







import os

import jsonlines
import numpy as np
from tqdm import tqdm


__lowerCAmelCase   =       2_0_4_8
__lowerCAmelCase   =       4_0_9_6
__lowerCAmelCase   =       4_2
__lowerCAmelCase   =       os.environ.pop("""PROCESS_TRAIN""", """false""")
__lowerCAmelCase   =       {"""null""": 0, """short""": 1, """long""": 2, """yes""": 3, """no""": 4}





def    UpperCAmelCase_						(__a :       Tuple					):






   """simple docstring"""

   def choose_first(__a :       List[Any]					,      __a :       Union[str, Any]=False					):
      assert isinstance(_snake_case					,      _snake_case					)
      if len(_snake_case					) == 1:
         _a							:      int     =  answer[0]
         return {k: [answer[k]] for k in answer} if is_long_answer else answer
      for a in answer:
         if is_long_answer:
            _a							:      Any     =  {k: [a[k]] for k in a}
         if len(a['start_token']					) > 0:
            break
      return a

   _a							:      int     =  {'id': example['id']}
   _a							:      List[Any]     =  example['annotations']
   _a							:      Optional[Any]     =  annotation['yes_no_answer']
   if 0 in yes_no_answer or 1 in yes_no_answer:
      _a							:      Any     =  ['yes'] if 1 in yes_no_answer else ['no']
      _a							:      str     =  []
      _a							:      List[Any]     =  []
      _a							:      str     =  ['<cls>']
   else:
      _a							:      List[str]     =  ['short']
      _a							:      int     =  choose_first(annotation['short_answers']					)
      if len(out['start_token']					) == 0:
         # answer will be long if short is not available
         _a							:      List[str]     =  ['long']
         _a							:      Any     =  choose_first(annotation['long_answer']					,      is_long_answer=_snake_case					)
         _a							:      Optional[int]     =  []
      answer.update(_snake_case					)

   # disregard some samples
   if len(answer['start_token']					) > 1 or answer["start_token"] == answer["end_token"]:
      _a							:      Optional[int]     =  True
   else:
      _a							:      List[str]     =  False

   _a							:      Dict     =  ['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
   if not all(isinstance(answer[k]					,      _snake_case					) for k in cols					):
      raise ValueError('Issue in ID'					,      example['id']					)

   return answer





def    UpperCAmelCase_						(__a :       int					,      __a :       Any=False					):






   """simple docstring"""

   _a							:      Any     =  _get_single_answer(_snake_case					)
   # bytes are of no use
   del answer["start_byte"]
   del answer["end_byte"]

   # handle yes_no answers explicitly
   if answer["category"][0] in ["yes", "no"]:  # category is list with one element
      _a							:      List[Any]     =  example['document']['tokens']
      _a							:      str     =  []
      for i in range(len(doc['token']					)					):
         if not doc["is_html"][i]:
            context.append(doc['token'][i]					)
      return {
          "context": " ".join(_snake_case					),
          "answer": {
              "start_token": -1_0_0,  # ignore index in cross-entropy
              "end_token": -1_0_0,  # ignore index in cross-entropy
              "category": answer["category"],
              "span": answer["category"],  # extra
          },
      }

   # later, help in removing all no answers
   if answer["start_token"] == [-1]:
      return {
          "context": "None",
          "answer": {
              "start_token": -1,
              "end_token": -1,
              "category": "null",
              "span": "None",  # extra
          },
      }

   # handling normal samples

   _a							:      Optional[int]     =  ['start_token', 'end_token']
   answer.update({k: answer[k][0] if len(answer[k]					) > 0 else answer[k] for k in cols}					)  # e.g. [10] == 10

   _a							:      Any     =  example['document']['tokens']
   _a							:      List[Any]     =  answer['start_token']
   _a							:      List[str]     =  answer['end_token']

   _a							:      Dict     =  []
   for i in range(len(doc['token']					)					):
      if not doc["is_html"][i]:
         context.append(doc['token'][i]					)
      else:
         if answer["start_token"] > i:
            start_token -= 1
         if answer["end_token"] > i:
            end_token -= 1
   _a							:      List[str]     =  ' '.join(context[start_token:end_token]					)

   # checking above code
   if assertion:
      _a							:      int     =  doc['is_html'][answer['start_token'] : answer['end_token']]
      _a							:      Optional[Any]     =  doc['token'][answer['start_token'] : answer['end_token']]
      _a							:      Union[str, Any]     =  ' '.join([old[i] for i in range(len(_snake_case					)					) if not is_html[i]]					)
      if new != old:
         print('ID:'					,      example['id']					)
         print('New:'					,      _snake_case					,      end='\n'					)
         print('Old:'					,      _snake_case					,      end='\n\n'					)

   return {
       "context": " ".join(_snake_case					),
       "answer": {
           "start_token": start_token,
           "end_token": end_token - 1,  # this makes it inclusive
           "category": answer["category"],  # either long or short
           "span": new,  # extra
       },
   }





def    UpperCAmelCase_						(__a :       Tuple					,      __a :       int					,      __a :       str=2_0_4_8					,      __a :       List[str]=4_0_9_6					,      __a :       str=True					):






   """simple docstring"""

   _a							:      int     =  get_context_and_ans(_snake_case					,      assertion=_snake_case					)
   _a							:      str     =  out['answer']

   # later, removing these samples
   if answer["start_token"] == -1:
      return {
          "example_id": example["id"],
          "input_ids": [[-1]],
          "labels": {
              "start_token": [-1],
              "end_token": [-1],
              "category": ["null"],
          },
      }

   _a							:      Optional[Any]     =  tokenizer(example['question']['text']					,      out['context']					).input_ids
   _a							:      List[Any]     =  input_ids.index(tokenizer.sep_token_id					) + 1

   # return yes/no
   if answer["category"][0] in ["yes", "no"]:  # category is list with one element
      _a							:      Tuple     =  []
      _a							:      Any     =  []
      _a							:      int     =  input_ids[:q_len]
      _a							:      Tuple     =  range(_snake_case					,      len(_snake_case					)					,      max_length - doc_stride					)
      for i in doc_start_indices:
         _a							:      str     =  i + max_length - q_len
         _a							:      Optional[Any]     =  input_ids[i:end_index]
         inputs.append(q_indices + slice					)
         category.append(answer['category'][0]					)
         if slice[-1] == tokenizer.sep_token_id:
            break

      return {
          "example_id": example["id"],
          "input_ids": inputs,
          "labels": {
              "start_token": [-1_0_0] * len(_snake_case					),
              "end_token": [-1_0_0] * len(_snake_case					),
              "category": category,
          },
      }

   _a							:      Optional[Any]     =  out['context'].split()
   _a							:      List[Any]     =  splitted_context[answer['end_token']]
   _a							:      Dict     =  len(
       tokenizer(
           ' '.join(splitted_context[: answer['start_token']]					)					,      add_special_tokens=_snake_case					,      ).input_ids					)
   _a							:      str     =  len(
       tokenizer(' '.join(splitted_context[: answer['end_token']]					)					,      add_special_tokens=_snake_case					).input_ids					)

   answer["start_token"] += q_len
   answer["end_token"] += q_len

   # fixing end token
   _a							:      Union[str, Any]     =  len(tokenizer(_snake_case					,      add_special_tokens=_snake_case					).input_ids					)
   if num_sub_tokens > 1:
      answer["end_token"] += num_sub_tokens - 1

   _a							:      Union[str, Any]     =  input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
   _a							:      Tuple     =  answer['start_token']
   _a							:      List[Any]     =  answer['end_token']

   if assertion:
      _a							:      Optional[int]     =  tokenizer.decode(_snake_case					)
      if answer["span"] != new:
         print('ISSUE IN TOKENIZATION'					)
         print('OLD:'					,      answer['span']					)
         print('NEW:'					,      _snake_case					,      end='\n\n'					)

   if len(_snake_case					) <= max_length:
      return {
          "example_id": example["id"],
          "input_ids": [input_ids],
          "labels": {
              "start_token": [answer["start_token"]],
              "end_token": [answer["end_token"]],
              "category": answer["category"],
          },
      }

   _a							:      List[str]     =  input_ids[:q_len]
   _a							:      Any     =  range(_snake_case					,      len(_snake_case					)					,      max_length - doc_stride					)

   _a							:      Optional[Any]     =  []
   _a							:      Any     =  []
   _a							:      Union[str, Any]     =  []
   _a							:      Optional[int]     =  []  # null, yes, no, long, short
   for i in doc_start_indices:
      _a							:      Union[str, Any]     =  i + max_length - q_len
      _a							:      Dict     =  input_ids[i:end_index]
      inputs.append(q_indices + slice					)
      assert len(inputs[-1]					) <= max_length, "Issue in truncating length"

      if start_token >= i and end_token <= end_index - 1:
         _a							:      Union[str, Any]     =  start_token - i + q_len
         _a							:      Optional[int]     =  end_token - i + q_len
         answers_category.append(answer['category'][0]					)  # ["short"] -> "short"
      else:
         _a							:      Union[str, Any]     =  -1_0_0
         _a							:      int     =  -1_0_0
         answers_category.append('null'					)
      _a							:      Any     =  inputs[-1][start_token : end_token + 1]

      answers_start_token.append(_snake_case					)
      answers_end_token.append(_snake_case					)
      if assertion:
         if new != old and new != [tokenizer.cls_token_id]:
            print('ISSUE in strided for ID:'					,      example['id']					)
            print('New:'					,      tokenizer.decode(_snake_case					)					)
            print('Old:'					,      tokenizer.decode(_snake_case					)					,      end='\n\n'					)
      if slice[-1] == tokenizer.sep_token_id:
         break

   return {
       "example_id": example["id"],
       "input_ids": inputs,
       "labels": {
           "start_token": answers_start_token,
           "end_token": answers_end_token,
           "category": answers_category,
       },
   }





def    UpperCAmelCase_						(__a :       Tuple					,      __a :       int					,      __a :       Optional[Any]=2_0_4_8					,      __a :       Optional[int]=4_0_9_6					,      __a :       Any=False					):






   """simple docstring"""

   _a							:      Union[str, Any]     =  get_strided_contexts_and_ans(
       _snake_case					,      _snake_case					,      doc_stride=_snake_case					,      max_length=_snake_case					,      assertion=_snake_case					,      )

   return example





def    UpperCAmelCase_						(__a :       List[Any]					,      __a :       List[Any]					):






   """simple docstring"""

   with jsonlines.open(_snake_case					,      'a'					) as writer:
      for example in tqdm(_snake_case					,      total=len(_snake_case					)					,      desc='Saving samples ... '					):
         _a							:      Optional[Any]     =  example['labels']
         for ids, start, end, cat in zip(
             example['input_ids']					,      labels['start_token']					,      labels['end_token']					,      labels['category']					,      ):
            if start == -1 and end == -1:
               continue  # leave waste samples with no answer
            if cat == "null" and np.random.rand() < 0.6:
               continue  # removing 50 % samples
            writer.write(
                {
                    'input_ids': ids,
                    'start_token': start,
                    'end_token': end,
                    'category': CATEGORY_MAPPING[cat],
                }					)


if __name__ == "__main__":
  from datasets import load_dataset

  from transformers import BigBirdTokenizer

  __lowerCAmelCase   =       load_dataset("""natural_questions""")
  __lowerCAmelCase   =       BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""")

  __lowerCAmelCase   =       data["""train""" if PROCESS_TRAIN == """true""" else """validation"""]

  __lowerCAmelCase   =       {
      """tokenizer""": tokenizer,
      """doc_stride""": DOC_STRIDE,
      """max_length""": MAX_LENGTH,
      """assertion""": False,
  }
  __lowerCAmelCase   =       data.map(prepare_inputs, fn_kwargs=fn_kwargs)
  __lowerCAmelCase   =       data.remove_columns(["""annotations""", """document""", """id""", """question"""])
  print(data)

  np.random.seed(SEED)
  __lowerCAmelCase   =       """nq-training.jsonl""" if PROCESS_TRAIN == """true""" else """nq-validation.jsonl"""
  save_to_disk(data, file_name=cache_file_name)





"""simple docstring"""



from __future__ import annotations

import queue





class 				a			:



      def __init__(   self     ,	UpperCamelCase_      ):
          UpperCAmelCase__	:      int     							=					data
          UpperCAmelCase__	:      Dict     							=					None
          UpperCAmelCase__	:      Optional[int]     							=					None




def 					lowerCamelCase   (     ):
    print('\n********Press N to stop entering at any point of time********\n' )
    UpperCAmelCase__	:      int     							=					input('Enter the value of the root node: ' ).strip().lower()
    UpperCAmelCase__	:      queue.Queue     							=					queue.Queue()
    UpperCAmelCase__	:      Dict     							=					TreeNode(int(_snake_case ) )
    q.put(_snake_case )
    while not q.empty():
        UpperCAmelCase__	:      Tuple     							=					q.get()
        UpperCAmelCase__	:      List[Any]     							=					F'''Enter the left node of {node_found.data}: '''
        UpperCAmelCase__	:      Dict     							=					input(_snake_case ).strip().lower() or 'n'
        if check == "n":
            return tree_node
        UpperCAmelCase__	:      Optional[int]     							=					TreeNode(int(_snake_case ) )
        UpperCAmelCase__	:      Tuple     							=					left_node
        q.put(_snake_case )
        UpperCAmelCase__	:      int     							=					F'''Enter the right node of {node_found.data}: '''
        UpperCAmelCase__	:      int     							=					input(_snake_case ).strip().lower() or 'n'
        if check == "n":
            return tree_node
        UpperCAmelCase__	:      int     							=					TreeNode(int(_snake_case ) )
        UpperCAmelCase__	:      Any     							=					right_node
        q.put(_snake_case )
    raise
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    print(node.data    ,end=',' )
    pre_order(node.left )
    pre_order(node.right )
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    in_order(node.left )
    print(node.data    ,end=',' )
    in_order(node.right )
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    post_order(node.left )
    post_order(node.right )
    print(node.data    ,end=',' )
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    UpperCAmelCase__	:      queue.Queue     							=					queue.Queue()
    q.put(_snake_case )
    while not q.empty():
        UpperCAmelCase__	:      int     							=					q.get()
        print(node_dequeued.data    ,end=',' )
        if node_dequeued.left:
            q.put(node_dequeued.left )
        if node_dequeued.right:
            q.put(node_dequeued.right )
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    UpperCAmelCase__	:      queue.Queue     							=					queue.Queue()
    q.put(_snake_case )
    while not q.empty():
        UpperCAmelCase__	:      Optional[Any]     							=					[]
        while not q.empty():
            UpperCAmelCase__	:      Union[str, Any]     							=					q.get()
            print(node_dequeued.data    ,end=',' )
            if node_dequeued.left:
                list_.append(node_dequeued.left )
            if node_dequeued.right:
                list_.append(node_dequeued.right )
        print()
        for node in list_:
            q.put(_snake_case )
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    UpperCAmelCase__	:      list[TreeNode]     							=					[]
    UpperCAmelCase__	:      Optional[int]     							=					node
    while n or stack:
        while n:  # start from root node, find its left child
            print(n.data    ,end=',' )
            stack.append(_snake_case )
            UpperCAmelCase__	:      Optional[int]     							=					n.left
        # end of while means current node doesn't have left child
        UpperCAmelCase__	:      Union[str, Any]     							=					stack.pop()
        # start to traverse its right child
        UpperCAmelCase__	:      Dict     							=					n.right
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    UpperCAmelCase__	:      list[TreeNode]     							=					[]
    UpperCAmelCase__	:      Optional[Any]     							=					node
    while n or stack:
        while n:
            stack.append(_snake_case )
            UpperCAmelCase__	:      int     							=					n.left
        UpperCAmelCase__	:      Tuple     							=					stack.pop()
        print(n.data    ,end=',' )
        UpperCAmelCase__	:      Tuple     							=					n.right
def 					lowerCamelCase   (     _snake_case ):
    if not isinstance(_snake_case    ,_snake_case ) or not node:
        return
    UpperCAmelCase__							,							UpperCAmelCase__	:      str     							=					[], []
    UpperCAmelCase__	:      Union[str, Any]     							=					node
    stacka.append(_snake_case )
    while stacka:  # to find the reversed order of post order, store it in stack2
        UpperCAmelCase__	:      Tuple     							=					stacka.pop()
        if n.left:
            stacka.append(n.left )
        if n.right:
            stacka.append(n.right )
        stacka.append(_snake_case )
    while stacka:  # pop up from stack2 will be the post order
        print(stacka.pop().data    ,end=',' )
def 					lowerCamelCase   (     _snake_case = ""    ,_snake_case=50    ,_snake_case="*" ):
    if not s:
        return "\n" + width * char
    UpperCAmelCase__							,							UpperCAmelCase__	:      str     							=					divmod(width - len(_snake_case ) - 2    ,2 )
    return F'''{left * char} {s} {(left + extra) * char}'''


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    print(prompt('Binary Tree Traversals'))

    UpperCamelCase__          =	build_tree()
    print(prompt('Pre Order Traversal'))
    pre_order(node)
    print(prompt() + '\n')

    print(prompt('In Order Traversal'))
    in_order(node)
    print(prompt() + '\n')

    print(prompt('Post Order Traversal'))
    post_order(node)
    print(prompt() + '\n')

    print(prompt('Level Order Traversal'))
    level_order(node)
    print(prompt() + '\n')

    print(prompt('Actual Level Order Traversal'))
    level_order_actual(node)
    print('*' * 50 + '\n')

    print(prompt('Pre Order Traversal - Iteration Version'))
    pre_order_iter(node)
    print(prompt() + '\n')

    print(prompt('In Order Traversal - Iteration Version'))
    in_order_iter(node)
    print(prompt() + '\n')

    print(prompt('Post Order Traversal - Iteration Version'))
    post_order_iter(node)
    print(prompt())

