'''simple docstring'''


import os

import jsonlines
import numpy as np
from tqdm import tqdm


snake_case__					:      List[Any]				      =  2048
snake_case__					:      Tuple				      =  4096
snake_case__					:      str				      =  42
snake_case__					:      List[str]				      =  os.environ.pop('''PROCESS_TRAIN''', '''false''')
snake_case__					:      Union[str, Any]				      =  {'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}


def  _lowerCamelCase   (				lowerCamelCase_ :							int       ):



     """simple docstring"""





     def choose_first(lowerCamelCase_ :							str     ,					lowerCamelCase_ :							Tuple=False       ):
          assert isinstance(lowerCamelCase_     ,					lowerCamelCase_       )
          if len(lowerCamelCase_       ) == 1:
               UpperCAmelCase_							:       Union[str, Any]       =				answer[0]
               return {k: [answer[k]] for k in answer} if is_long_answer else answer
          for a in answer:
               if is_long_answer:
                    UpperCAmelCase_							:       str       =				{k: [a[k]] for k in a}
               if len(a['start_token']       ) > 0:
                    break
          return a

     UpperCAmelCase_							:       Optional[int]       =				{'id': example['id']}
     UpperCAmelCase_							:       Optional[int]       =				example['annotations']
     UpperCAmelCase_							:       Dict       =				annotation['yes_no_answer']
     if 0 in yes_no_answer or 1 in yes_no_answer:
          UpperCAmelCase_							:       Optional[Any]       =				['yes'] if 1 in yes_no_answer else ['no']
          UpperCAmelCase_							:       Optional[Any]       =				[]
          UpperCAmelCase_							:       int       =				[]
          UpperCAmelCase_							:       Union[str, Any]       =				['<cls>']
     else:
          UpperCAmelCase_							:       Tuple       =				['short']
          UpperCAmelCase_							:       Dict       =				choose_first(annotation['short_answers']       )
          if len(out['start_token']       ) == 0:
               # answer will be long if short is not available
               UpperCAmelCase_							:       str       =				['long']
               UpperCAmelCase_							:       Union[str, Any]       =				choose_first(annotation['long_answer']     ,					is_long_answer=lowerCamelCase_       )
               UpperCAmelCase_							:       int       =				[]
          answer.update(lowerCamelCase_       )

     # disregard some samples
     if len(answer['start_token']       ) > 1 or answer["start_token"] == answer["end_token"]:
          UpperCAmelCase_							:       List[Any]       =				True
     else:
          UpperCAmelCase_							:       List[Any]       =				False

     UpperCAmelCase_							:       Tuple       =				['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
     if not all(isinstance(answer[k]     ,					lowerCamelCase_       ) for k in cols       ):
          raise ValueError('Issue in ID'     ,					example['id']       )

     return answer


def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[int]     ,					lowerCamelCase_ :							List[str]=False       ):



     """simple docstring"""





     UpperCAmelCase_							:       Any       =				_get_single_answer(lowerCamelCase_       )
     # bytes are of no use
     del answer["start_byte"]
     del answer["end_byte"]

     # handle yes_no answers explicitly
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          UpperCAmelCase_							:       int       =				example['document']['tokens']
          UpperCAmelCase_							:       Any       =				[]
          for i in range(len(doc['token']       )       ):
               if not doc["is_html"][i]:
                    context.append(doc['token'][i]       )
          return {
              "context": " ".join(lowerCamelCase_       ),
              "answer": {
                  "start_token": -100,  # ignore index in cross-entropy
                  "end_token": -100,  # ignore index in cross-entropy
                  "category": answer["category"],
                  "span": answer["category"],  # extra
              },
          }

     # later, help in removing all no answers
     if answer["start_token"] == [-1]:
          return {
              "context": "None",
              "answer": {
                  "start_token": -1,
                  "end_token": -1,
                  "category": "null",
                  "span": "None",  # extra
              },
          }

     # handling normal samples

     UpperCAmelCase_							:       int       =				['start_token', 'end_token']
     answer.update({k: answer[k][0] if len(answer[k]       ) > 0 else answer[k] for k in cols}       )  # e.g. [10] == 10

     UpperCAmelCase_							:       Union[str, Any]       =				example['document']['tokens']
     UpperCAmelCase_							:       Optional[int]       =				answer['start_token']
     UpperCAmelCase_							:       str       =				answer['end_token']

     UpperCAmelCase_							:       List[Any]       =				[]
     for i in range(len(doc['token']       )       ):
          if not doc["is_html"][i]:
               context.append(doc['token'][i]       )
          else:
               if answer["start_token"] > i:
                    start_token -= 1
               if answer["end_token"] > i:
                    end_token -= 1
     UpperCAmelCase_							:       Dict       =				' '.join(context[start_token:end_token]       )

     # checking above code
     if assertion:
          UpperCAmelCase_							:       Dict       =				doc['is_html'][answer['start_token'] : answer['end_token']]
          UpperCAmelCase_							:       Union[str, Any]       =				doc['token'][answer['start_token'] : answer['end_token']]
          UpperCAmelCase_							:       Optional[Any]       =				' '.join([old[i] for i in range(len(lowerCamelCase_       )       ) if not is_html[i]]       )
          if new != old:
               print('ID:'     ,					example['id']       )
               print('New:'     ,					lowerCamelCase_     ,					end='\n'       )
               print('Old:'     ,					lowerCamelCase_     ,					end='\n\n'       )

     return {
         "context": " ".join(lowerCamelCase_       ),
         "answer": {
             "start_token": start_token,
             "end_token": end_token - 1,  # this makes it inclusive
             "category": answer["category"],  # either long or short
             "span": new,  # extra
         },
     }


def  _lowerCamelCase   (				lowerCamelCase_ :							str     ,					lowerCamelCase_ :							str     ,					lowerCamelCase_ :							List[str]=2048     ,					lowerCamelCase_ :							List[str]=4096     ,					lowerCamelCase_ :							List[Any]=True       ):



     """simple docstring"""





     UpperCAmelCase_							:       str       =				get_context_and_ans(lowerCamelCase_     ,					assertion=lowerCamelCase_       )
     UpperCAmelCase_							:       Optional[int]       =				out['answer']

     # later, removing these samples
     if answer["start_token"] == -1:
          return {
              "example_id": example["id"],
              "input_ids": [[-1]],
              "labels": {
                  "start_token": [-1],
                  "end_token": [-1],
                  "category": ["null"],
              },
          }

     UpperCAmelCase_							:       Optional[int]       =				tokenizer(example['question']['text']     ,					out['context']       ).input_ids
     UpperCAmelCase_							:       Dict       =				input_ids.index(tokenizer.sep_token_id       ) + 1

     # return yes/no
     if answer["category"][0] in ["yes", "no"]:  # category is list with one element
          UpperCAmelCase_							:       int       =				[]
          UpperCAmelCase_							:       str       =				[]
          UpperCAmelCase_							:       Tuple       =				input_ids[:q_len]
          UpperCAmelCase_							:       List[Any]       =				range(lowerCamelCase_     ,					len(lowerCamelCase_       )     ,					max_length - doc_stride       )
          for i in doc_start_indices:
               UpperCAmelCase_							:       int       =				i + max_length - q_len
               UpperCAmelCase_							:       List[Any]       =				input_ids[i:end_index]
               inputs.append(q_indices + slice       )
               category.append(answer['category'][0]       )
               if slice[-1] == tokenizer.sep_token_id:
                    break

          return {
              "example_id": example["id"],
              "input_ids": inputs,
              "labels": {
                  "start_token": [-100] * len(lowerCamelCase_       ),
                  "end_token": [-100] * len(lowerCamelCase_       ),
                  "category": category,
              },
          }

     UpperCAmelCase_							:       Union[str, Any]       =				out['context'].split()
     UpperCAmelCase_							:       Tuple       =				splitted_context[answer['end_token']]
     UpperCAmelCase_							:       Any       =				len(
         tokenizer(
             ' '.join(splitted_context[: answer['start_token']]       )     ,					add_special_tokens=lowerCamelCase_     ,					).input_ids       )
     UpperCAmelCase_							:       Union[str, Any]       =				len(
         tokenizer(' '.join(splitted_context[: answer['end_token']]       )     ,					add_special_tokens=lowerCamelCase_       ).input_ids       )

     answer["start_token"] += q_len
     answer["end_token"] += q_len

     # fixing end token
     UpperCAmelCase_							:       int       =				len(tokenizer(lowerCamelCase_     ,					add_special_tokens=lowerCamelCase_       ).input_ids       )
     if num_sub_tokens > 1:
          answer["end_token"] += num_sub_tokens - 1

     UpperCAmelCase_							:       str       =				input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
     UpperCAmelCase_							:       int       =				answer['start_token']
     UpperCAmelCase_							:       str       =				answer['end_token']

     if assertion:
          UpperCAmelCase_							:       int       =				tokenizer.decode(lowerCamelCase_       )
          if answer["span"] != new:
               print('ISSUE IN TOKENIZATION'       )
               print('OLD:'     ,					answer['span']       )
               print('NEW:'     ,					lowerCamelCase_     ,					end='\n\n'       )

     if len(lowerCamelCase_       ) <= max_length:
          return {
              "example_id": example["id"],
              "input_ids": [input_ids],
              "labels": {
                  "start_token": [answer["start_token"]],
                  "end_token": [answer["end_token"]],
                  "category": answer["category"],
              },
          }

     UpperCAmelCase_							:       Any       =				input_ids[:q_len]
     UpperCAmelCase_							:       Optional[int]       =				range(lowerCamelCase_     ,					len(lowerCamelCase_       )     ,					max_length - doc_stride       )

     UpperCAmelCase_							:       Dict       =				[]
     UpperCAmelCase_							:       Union[str, Any]       =				[]
     UpperCAmelCase_							:       List[Any]       =				[]
     UpperCAmelCase_							:       str       =				[]  # null, yes, no, long, short
     for i in doc_start_indices:
          UpperCAmelCase_							:       Union[str, Any]       =				i + max_length - q_len
          UpperCAmelCase_							:       str       =				input_ids[i:end_index]
          inputs.append(q_indices + slice       )
          assert len(inputs[-1]       ) <= max_length, "Issue in truncating length"

          if start_token >= i and end_token <= end_index - 1:
               UpperCAmelCase_							:       Any       =				start_token - i + q_len
               UpperCAmelCase_							:       Optional[int]       =				end_token - i + q_len
               answers_category.append(answer['category'][0]       )  # ["short"] -> "short"
          else:
               UpperCAmelCase_							:       Tuple       =				-100
               UpperCAmelCase_							:       Tuple       =				-100
               answers_category.append('null'       )
          UpperCAmelCase_							:       Dict       =				inputs[-1][start_token : end_token + 1]

          answers_start_token.append(lowerCamelCase_       )
          answers_end_token.append(lowerCamelCase_       )
          if assertion:
               if new != old and new != [tokenizer.cls_token_id]:
                    print('ISSUE in strided for ID:'     ,					example['id']       )
                    print('New:'     ,					tokenizer.decode(lowerCamelCase_       )       )
                    print('Old:'     ,					tokenizer.decode(lowerCamelCase_       )     ,					end='\n\n'       )
          if slice[-1] == tokenizer.sep_token_id:
               break

     return {
         "example_id": example["id"],
         "input_ids": inputs,
         "labels": {
             "start_token": answers_start_token,
             "end_token": answers_end_token,
             "category": answers_category,
         },
     }


def  _lowerCamelCase   (				lowerCamelCase_ :							Tuple     ,					lowerCamelCase_ :							Tuple     ,					lowerCamelCase_ :							Union[str, Any]=2048     ,					lowerCamelCase_ :							str=4096     ,					lowerCamelCase_ :							Tuple=False       ):



     """simple docstring"""





     UpperCAmelCase_							:       int       =				get_strided_contexts_and_ans(
         lowerCamelCase_     ,					lowerCamelCase_     ,					doc_stride=lowerCamelCase_     ,					max_length=lowerCamelCase_     ,					assertion=lowerCamelCase_     ,					)

     return example


def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[Any]     ,					lowerCamelCase_ :							Any       ):



     """simple docstring"""





     with jsonlines.open(lowerCamelCase_     ,					'a'       ) as writer:
          for example in tqdm(lowerCamelCase_     ,					total=len(lowerCamelCase_       )     ,					desc='Saving samples ... '       ):
               UpperCAmelCase_							:       List[str]       =				example['labels']
               for ids, start, end, cat in zip(
                   example['input_ids']     ,					labels['start_token']     ,					labels['end_token']     ,					labels['category']     ,					):
                    if start == -1 and end == -1:
                         continue  # leave waste samples with no answer
                    if cat == "null" and np.random.rand() < 0.6:
                         continue  # removing 50 % samples
                    writer.write(
                        {
                            'input_ids': ids,
                            'start_token': start,
                            'end_token': end,
                            'category': CATEGORY_MAPPING[cat],
                        }       )


if __name__ == "__main__":
 from datasets import load_dataset

 from transformers import BigBirdTokenizer

 snake_case__					:      List[str]				      =  load_dataset('''natural_questions''')
 snake_case__					:      List[Any]				      =  BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

 snake_case__					:      List[str]				      =  data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

 snake_case__					:      Dict				      =  {
     '''tokenizer''': tokenizer,
     '''doc_stride''': DOC_STRIDE,
     '''max_length''': MAX_LENGTH,
     '''assertion''': False,
 }
 snake_case__					:      Optional[Any]				      =  data.map(prepare_inputs, fn_kwargs=fn_kwargs)
 snake_case__					:      str				      =  data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
 print(data)

 np.random.seed(SEED)
 snake_case__					:      Any				      =  '''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
 save_to_disk(data, file_name=cache_file_name)







'''simple docstring'''


from manim import *






class 					__SCREAMING_SNAKE_CASE						(	lowerCamelCase_	):







  '''simple docstring'''






  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Dict       =				Rectangle(height=0.5				,		width=0.5						)
       UpperCAmelCase_							:       Any       =				Rectangle(height=0.46				,		width=0.46						).set_stroke(width=0						)

       UpperCAmelCase_							:       List[str]       =				Rectangle(height=0.25				,		width=0.25						)

       UpperCAmelCase_							:       Any       =				[mem.copy() for i in range(6						)]
       UpperCAmelCase_							:       Union[str, Any]       =				[mem.copy() for i in range(6						)]
       UpperCAmelCase_							:       Optional[int]       =				VGroup(*snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       Any       =				VGroup(*snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       str       =				VGroup(snake_case_				,		snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       Any       =				Text('CPU'				,		font_size=2_4						)
       UpperCAmelCase_							:       Tuple       =				Group(snake_case_				,		snake_case_						).arrange(snake_case_				,		buff=0.5				,		aligned_edge=snake_case_						)
       cpu.move_to([-2.5, -0.5, 0]						)
       self.add(snake_case_						)

       UpperCAmelCase_							:       str       =				[mem.copy() for i in range(4						)]
       UpperCAmelCase_							:       Dict       =				VGroup(*snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       List[str]       =				Text('GPU'				,		font_size=2_4						)
       UpperCAmelCase_							:       Optional[int]       =				Group(snake_case_				,		snake_case_						).arrange(snake_case_				,		buff=0.5				,		aligned_edge=snake_case_						)
       gpu.move_to([-1, -1, 0]						)
       self.add(snake_case_						)

       UpperCAmelCase_							:       str       =				[mem.copy() for i in range(6						)]
       UpperCAmelCase_							:       Tuple       =				VGroup(*snake_case_						).arrange(snake_case_				,		buff=0						)

       UpperCAmelCase_							:       str       =				Text('Model'				,		font_size=2_4						)
       UpperCAmelCase_							:       Optional[int]       =				Group(snake_case_				,		snake_case_						).arrange(snake_case_				,		buff=0.5				,		aligned_edge=snake_case_						)
       model.move_to([3, -1.0, 0]						)
       self.add(snake_case_						)

       UpperCAmelCase_							:       str       =				[]
       UpperCAmelCase_							:       Optional[Any]       =				[]

       for i, rect in enumerate(snake_case_						):
            UpperCAmelCase_							:       str       =				fill.copy().set_fill(snake_case_				,		opacity=0.8						)
            target.move_to(snake_case_						)
            model_arr.append(snake_case_						)

            UpperCAmelCase_							:       int       =				Rectangle(height=0.46				,		width=0.46						).set_stroke(width=0.0						).set_fill(snake_case_				,		opacity=0.8						)
            cpu_target.move_to(cpu_left_col_base[i]						)
            model_cpu_arr.append(snake_case_						)

       self.add(*snake_case_				,		*snake_case_						)

       UpperCAmelCase_							:       List[Any]       =				[meta_mem.copy() for i in range(6						)]
       UpperCAmelCase_							:       List[str]       =				[meta_mem.copy() for i in range(6						)]
       UpperCAmelCase_							:       Tuple       =				VGroup(*snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       Dict       =				VGroup(*snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       Optional[Any]       =				VGroup(snake_case_				,		snake_case_						).arrange(snake_case_				,		buff=0						)
       UpperCAmelCase_							:       Tuple       =				Text('Disk'				,		font_size=2_4						)
       UpperCAmelCase_							:       Union[str, Any]       =				Group(snake_case_				,		snake_case_						).arrange(snake_case_				,		buff=0.5				,		aligned_edge=snake_case_						)
       disk.move_to([-4, -1.25, 0]						)
       self.add(snake_case_				,		snake_case_						)

       UpperCAmelCase_							:       List[Any]       =				Square(side_length=2.2						)
       key.move_to([-5, 2, 0]						)

       UpperCAmelCase_							:       Any       =				MarkupText(
           F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''				,		font_size=1_8				,		)

       key_text.move_to([-5, 2.4, 0]						)

       self.add(snake_case_				,		snake_case_						)

       UpperCAmelCase_							:       Dict       =				MarkupText(
           F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''				,		font_size=1_8				,		)

       blue_text.next_to(snake_case_				,		DOWN * 2.4				,		aligned_edge=key_text.get_left()						)
       self.add(snake_case_						)

       UpperCAmelCase_							:       Optional[Any]       =				MarkupText(
           F'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.'''				,		font_size=2_4				,		)
       step_a.move_to([2, 2, 0]						)

       self.play(Write(snake_case_						)						)

       UpperCAmelCase_							:       Tuple       =				Square(0.3						)
       input.set_fill(snake_case_				,		opacity=1.0						)
       input.set_stroke(width=0.0						)
       input.next_to(model_base[0]				,		snake_case_				,		buff=0.5						)

       self.play(Write(snake_case_						)						)

       input.generate_target()
       input.target.next_to(model_arr[0]				,		direction=snake_case_				,		buff=0.02						)
       self.play(MoveToTarget(snake_case_						)						)

       self.play(FadeOut(snake_case_						)						)

       UpperCAmelCase_							:       Any       =				Arrow(start=snake_case_				,		end=snake_case_				,		color=snake_case_				,		buff=0.5						)
       a.next_to(model_arr[0].get_left()				,		snake_case_				,		buff=0.2						)

       model_cpu_arr[0].generate_target()
       model_cpu_arr[0].target.move_to(gpu_rect[0]						)

       UpperCAmelCase_							:       List[str]       =				MarkupText(
           F'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'''				,		font_size=2_4				,		)
       step_a.move_to([2, 2, 0]						)

       self.play(Write(snake_case_				,		run_time=3						)						)

       UpperCAmelCase_							:       List[Any]       =				{'run_time': 1, 'fade_in': True, 'fade_out': True, 'buff': 0.02}

       self.play(
           Write(snake_case_						)				,		Circumscribe(model_arr[0]				,		color=snake_case_				,		**snake_case_						)				,		Circumscribe(model_cpu_arr[0]				,		color=snake_case_				,		**snake_case_						)				,		Circumscribe(gpu_rect[0]				,		color=snake_case_				,		**snake_case_						)				,		)
       self.play(MoveToTarget(model_cpu_arr[0]						)						)

       UpperCAmelCase_							:       Union[str, Any]       =				a.copy()
       for i in range(6						):
            a_c.next_to(model_arr[i].get_right() + 0.02				,		snake_case_				,		buff=0.2						)

            input.generate_target()
            input.target.move_to(model_arr[i].get_right() + 0.02						)

            UpperCAmelCase_							:       Tuple       =				AnimationGroup(
                FadeOut(snake_case_				,		run_time=0.5						)				,		MoveToTarget(snake_case_				,		run_time=0.5						)				,		FadeIn(snake_case_				,		run_time=0.5						)				,		lag_ratio=0.2						)

            self.play(snake_case_						)

            model_cpu_arr[i].generate_target()
            model_cpu_arr[i].target.move_to(cpu_left_col_base[i]						)

            if i < 5:
                 model_cpu_arr[i + 1].generate_target()
                 model_cpu_arr[i + 1].target.move_to(gpu_rect[0]						)
                 if i >= 1:
                      UpperCAmelCase_							:       Any       =				0.7

                 self.play(
                     Circumscribe(model_arr[i]				,		**snake_case_						)				,		Circumscribe(cpu_left_col_base[i]				,		**snake_case_						)				,		Circumscribe(cpu_left_col_base[i + 1]				,		color=snake_case_				,		**snake_case_						)				,		Circumscribe(gpu_rect[0]				,		color=snake_case_				,		**snake_case_						)				,		Circumscribe(model_arr[i + 1]				,		color=snake_case_				,		**snake_case_						)				,		)
                 if i < 1:
                      self.play(
                          MoveToTarget(model_cpu_arr[i]						)				,		MoveToTarget(model_cpu_arr[i + 1]						)				,		)
                 else:
                      self.play(
                          MoveToTarget(model_cpu_arr[i]				,		run_time=0.7						)				,		MoveToTarget(model_cpu_arr[i + 1]				,		run_time=0.7						)				,		)
            else:
                 model_cpu_arr[i].generate_target()
                 model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]						)
                 input.generate_target()
                 input.target.next_to(model_arr[-1].get_right()				,		RIGHT + 0.02				,		buff=0.2						)

                 self.play(
                     Circumscribe(model_arr[-1]				,		color=snake_case_				,		**snake_case_						)				,		Circumscribe(cpu_left_col_base[-1]				,		color=snake_case_				,		**snake_case_						)				,		Circumscribe(gpu_rect[0]				,		color=snake_case_				,		**snake_case_						)				,		)

                 self.play(MoveToTarget(model_cpu_arr[i]						)						)

            UpperCAmelCase_							:       Any       =				a_c
            UpperCAmelCase_							:       int       =				a_c.copy()

       input.generate_target()
       input.target.next_to(model_base[-1]				,		RIGHT + 0.02				,		buff=0.5						)
       self.play(
           FadeOut(snake_case_						)				,		FadeOut(snake_case_				,		run_time=0.5						)				,		)

       UpperCAmelCase_							:       Optional[Any]       =				MarkupText(F'''Inference on a model too large for GPU memory\nis successfully completed.'''				,		font_size=2_4						)
       step_a.move_to([2, 2, 0]						)

       self.play(Write(snake_case_				,		run_time=3						)				,		MoveToTarget(snake_case_						)						)

       self.wait()






