




'''simple docstring'''







import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowerCAmelCase_     						=				2_0_4_8
lowerCAmelCase_     						=				4_0_9_6
lowerCAmelCase_     						=				4_2
lowerCAmelCase_     						=				os.environ.pop('''PROCESS_TRAIN''', '''false''')
lowerCAmelCase_     						=				{'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   def choose_first(UpperCAmelCase      ,UpperCAmelCase=False						):
      assert isinstance(UpperCAmelCase      ,UpperCAmelCase						)
      if len(UpperCAmelCase						) == 1:
         A__       =     answer[0]
         return {k: [answer[k]] for k in answer} if is_long_answer else answer
      for a in answer:
         if is_long_answer:
            A__       =     {k: [a[k]] for k in a}
         if len(a['start_token']						) > 0:
            break
      return a

   A__       =     {'id': example['id']}
   A__       =     example['annotations']
   A__       =     annotation['yes_no_answer']
   if 0 in yes_no_answer or 1 in yes_no_answer:
      A__       =     ['yes'] if 1 in yes_no_answer else ['no']
      A__       =     A__       =     []
      A__       =     A__       =     []
      A__       =     ['<cls>']
   else:
      A__       =     ['short']
      A__       =     choose_first(annotation['short_answers']						)
      if len(out['start_token']						) == 0:
         # answer will be long if short is not available
         A__       =     ['long']
         A__       =     choose_first(annotation['long_answer']      ,is_long_answer=UpperCAmelCase						)
         A__       =     []
      answer.update(UpperCAmelCase						)

   # disregard some samples
   if len(answer['start_token']						) > 1 or answer["start_token"] == answer["end_token"]:
      A__       =     True
   else:
      A__       =     False

   A__       =     ['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
   if not all(isinstance(answer[k]      ,UpperCAmelCase						) for k in cols						):
      raise ValueError('Issue in ID'      ,example['id']						)

   return answer







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase=False						):






   '''simple docstring'''







   A__       =     _get_single_answer(UpperCAmelCase						)
   # bytes are of no use
   del answer["start_byte"]
   del answer["end_byte"]

   # handle yes_no answers explicitly
   if answer["category"][0] in ["yes", "no"]:  # category is list with one element
      A__       =     example['document']['tokens']
      A__       =     []
      for i in range(len(doc['token']						)						):
         if not doc["is_html"][i]:
            context.append(doc['token'][i]						)
      return {
          "context": " ".join(UpperCAmelCase						),
          "answer": {
              "start_token": -100,  # ignore index in cross-entropy
              "end_token": -100,  # ignore index in cross-entropy
              "category": answer["category"],
              "span": answer["category"],  # extra
          },
      }

   # later, help in removing all no answers
   if answer["start_token"] == [-1]:
      return {
          "context": "None",
          "answer": {
              "start_token": -1,
              "end_token": -1,
              "category": "null",
              "span": "None",  # extra
          },
      }

   # handling normal samples

   A__       =     ['start_token', 'end_token']
   answer.update({k: answer[k][0] if len(answer[k]						) > 0 else answer[k] for k in cols}						)  # e.g. [10] == 10

   A__       =     example['document']['tokens']
   A__       =     answer['start_token']
   A__       =     answer['end_token']

   A__       =     []
   for i in range(len(doc['token']						)						):
      if not doc["is_html"][i]:
         context.append(doc['token'][i]						)
      else:
         if answer["start_token"] > i:
            start_token -= 1
         if answer["end_token"] > i:
            end_token -= 1
   A__       =     ' '.join(context[start_token:end_token]						)

   # checking above code
   if assertion:
      A__       =     doc['is_html'][answer['start_token'] : answer['end_token']]
      A__       =     doc['token'][answer['start_token'] : answer['end_token']]
      A__       =     ' '.join([old[i] for i in range(len(UpperCAmelCase						)						) if not is_html[i]]						)
      if new != old:
         print('ID:'      ,example['id']						)
         print('New:'      ,UpperCAmelCase      ,end='\n'						)
         print('Old:'      ,UpperCAmelCase      ,end='\n\n'						)

   return {
       "context": " ".join(UpperCAmelCase						),
       "answer": {
           "start_token": start_token,
           "end_token": end_token - 1,  # this makes it inclusive
           "category": answer["category"],  # either long or short
           "span": new,  # extra
       },
   }







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=2048      ,UpperCAmelCase=4096      ,UpperCAmelCase=True						):






   '''simple docstring'''







   A__       =     get_context_and_ans(UpperCAmelCase      ,assertion=UpperCAmelCase						)
   A__       =     out['answer']

   # later, removing these samples
   if answer["start_token"] == -1:
      return {
          "example_id": example["id"],
          "input_ids": [[-1]],
          "labels": {
              "start_token": [-1],
              "end_token": [-1],
              "category": ["null"],
          },
      }

   A__       =     tokenizer(example['question']['text']      ,out['context']						).input_ids
   A__       =     input_ids.index(tokenizer.sep_token_id						) + 1

   # return yes/no
   if answer["category"][0] in ["yes", "no"]:  # category is list with one element
      A__       =     []
      A__       =     []
      A__       =     input_ids[:q_len]
      A__       =     range(UpperCAmelCase      ,len(UpperCAmelCase						)      ,max_length - doc_stride						)
      for i in doc_start_indices:
         A__       =     i + max_length - q_len
         A__       =     input_ids[i:end_index]
         inputs.append(q_indices + slice						)
         category.append(answer['category'][0]						)
         if slice[-1] == tokenizer.sep_token_id:
            break

      return {
          "example_id": example["id"],
          "input_ids": inputs,
          "labels": {
              "start_token": [-100] * len(UpperCAmelCase						),
              "end_token": [-100] * len(UpperCAmelCase						),
              "category": category,
          },
      }

   A__       =     out['context'].split()
   A__       =     splitted_context[answer['end_token']]
   A__       =     len(
       tokenizer(
           ' '.join(splitted_context[: answer['start_token']]						)      ,add_special_tokens=UpperCAmelCase      ,).input_ids						)
   A__       =     len(
       tokenizer(' '.join(splitted_context[: answer['end_token']]						)      ,add_special_tokens=UpperCAmelCase						).input_ids						)

   answer["start_token"] += q_len
   answer["end_token"] += q_len

   # fixing end token
   A__       =     len(tokenizer(UpperCAmelCase      ,add_special_tokens=UpperCAmelCase						).input_ids						)
   if num_sub_tokens > 1:
      answer["end_token"] += num_sub_tokens - 1

   A__       =     input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
   A__       =     answer['start_token']
   A__       =     answer['end_token']

   if assertion:
      A__       =     tokenizer.decode(UpperCAmelCase						)
      if answer["span"] != new:
         print('ISSUE IN TOKENIZATION'						)
         print('OLD:'      ,answer['span']						)
         print('NEW:'      ,UpperCAmelCase      ,end='\n\n'						)

   if len(UpperCAmelCase						) <= max_length:
      return {
          "example_id": example["id"],
          "input_ids": [input_ids],
          "labels": {
              "start_token": [answer["start_token"]],
              "end_token": [answer["end_token"]],
              "category": answer["category"],
          },
      }

   A__       =     input_ids[:q_len]
   A__       =     range(UpperCAmelCase      ,len(UpperCAmelCase						)      ,max_length - doc_stride						)

   A__       =     []
   A__       =     []
   A__       =     []
   A__       =     []  # null, yes, no, long, short
   for i in doc_start_indices:
      A__       =     i + max_length - q_len
      A__       =     input_ids[i:end_index]
      inputs.append(q_indices + slice						)
      assert len(inputs[-1]						) <= max_length, "Issue in truncating length"

      if start_token >= i and end_token <= end_index - 1:
         A__       =     start_token - i + q_len
         A__       =     end_token - i + q_len
         answers_category.append(answer['category'][0]						)  # ["short"] -> "short"
      else:
         A__       =     -100
         A__       =     -100
         answers_category.append('null'						)
      A__       =     inputs[-1][start_token : end_token + 1]

      answers_start_token.append(UpperCAmelCase						)
      answers_end_token.append(UpperCAmelCase						)
      if assertion:
         if new != old and new != [tokenizer.cls_token_id]:
            print('ISSUE in strided for ID:'      ,example['id']						)
            print('New:'      ,tokenizer.decode(UpperCAmelCase						)						)
            print('Old:'      ,tokenizer.decode(UpperCAmelCase						)      ,end='\n\n'						)
      if slice[-1] == tokenizer.sep_token_id:
         break

   return {
       "example_id": example["id"],
       "input_ids": inputs,
       "labels": {
           "start_token": answers_start_token,
           "end_token": answers_end_token,
           "category": answers_category,
       },
   }







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=2048      ,UpperCAmelCase=4096      ,UpperCAmelCase=False						):






   '''simple docstring'''







   A__       =     get_strided_contexts_and_ans(
       UpperCAmelCase      ,UpperCAmelCase      ,doc_stride=UpperCAmelCase      ,max_length=UpperCAmelCase      ,assertion=UpperCAmelCase      ,)

   return example







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   with jsonlines.open(UpperCAmelCase      ,'a'						) as writer:
      for example in tqdm(UpperCAmelCase      ,total=len(UpperCAmelCase						)      ,desc='Saving samples ... '						):
         A__       =     example['labels']
         for ids, start, end, cat in zip(
             example['input_ids']      ,labels['start_token']      ,labels['end_token']      ,labels['category']      ,):
            if start == -1 and end == -1:
               continue  # leave waste samples with no answer
            if cat == "null" and np.random.rand() < 0.6:
               continue  # removing 50 % samples
            writer.write(
                {
                    'input_ids': ids,
                    'start_token': start,
                    'end_token': end,
                    'category': CATEGORY_MAPPING[cat],
                }						)


if __name__ == "__main__":
    from datasets import load_dataset

    from transformers import BigBirdTokenizer

    lowerCAmelCase_     						=				load_dataset('''natural_questions''')
    lowerCAmelCase_     						=				BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

    lowerCAmelCase_     						=				data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

    lowerCAmelCase_     						=				{
        '''tokenizer''': tokenizer,
        '''doc_stride''': DOC_STRIDE,
        '''max_length''': MAX_LENGTH,
        '''assertion''': False,
    }
    lowerCAmelCase_     						=				data.map(prepare_inputs, fn_kwargs=fn_kwargs)
    lowerCAmelCase_     						=				data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
    print(data)

    np.random.seed(SEED)
    lowerCAmelCase_     						=				'''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
    save_to_disk(data, file_name=cache_file_name)








'''simple docstring'''







import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  List[str]    )       ->   str:






    """simple docstring"""


    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()






 @property
 def 						_UpperCamelCase (self       :  int    )       ->   Any:






    """simple docstring"""


    A__       =     1
    A__       =     3
    A__       =     (32, 32)

    A__       =     floats_tensor((batch_size, num_channels) + sizes    ,		rng=random.Random(0    )    ).to(a    )
    return image






 @property
 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Tuple:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     UNetaDConditionModel(
        block_out_channels=(32, 32, 64)    ,		layers_per_block=2    ,		sample_size=32    ,		in_channels=7    ,		out_channels=4    ,		down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D', 'CrossAttnDownBlock2D')    ,		up_block_types=('CrossAttnUpBlock2D', 'CrossAttnUpBlock2D', 'UpBlock2D')    ,		cross_attention_dim=32    ,		attention_head_dim=8    ,		use_linear_projection=a    ,		only_cross_attention=(True, True, False)    ,		num_class_embeds=1_00    ,		)
    return model






 @property
 def 						_UpperCamelCase (self       :  Tuple    )       ->   Optional[Any]:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     AutoencoderKL(
        block_out_channels=[32, 32, 64]    ,		in_channels=3    ,		out_channels=3    ,		down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D', 'DownEncoderBlock2D']    ,		up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D', 'UpDecoderBlock2D']    ,		latent_channels=4    ,		)
    return model






 @property
 def 						_UpperCamelCase (self       :  str    )       ->   List[str]:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     CLIPTextConfig(
        bos_token_id=0    ,		eos_token_id=2    ,		hidden_size=32    ,		intermediate_size=37    ,		layer_norm_eps=1e-05    ,		num_attention_heads=4    ,		num_hidden_layers=5    ,		pad_token_id=1    ,		vocab_size=10_00    ,		hidden_act='gelu'    ,		projection_dim=5_12    ,		)
    return CLIPTextModel(a    )






 def 						_UpperCamelCase (self       :  str    )       ->   List[str]:






    """simple docstring"""


    A__       =     'cpu'  # ensure determinism for the device-dependent torch.Generator
    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)

    A__       =     output.images

    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		return_dict=a    ,		)[0]

    A__       =     image[0, -3:, -3:, -1]
    A__       =     image_from_tuple[0, -3:, -3:, -1]

    A__       =     low_res_image.size[0] * 4
    assert image.shape == (1, expected_height_width, expected_height_width, 3)
    A__       =     np.array([0.3113, 0.3910, 0.4272, 0.4859, 0.5061, 0.4652, 0.5362, 0.5715, 0.5661]    )

    assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1e-2
    assert np.abs(image_from_tuple_slice.flatten() - expected_slice    ).max() < 1e-2






 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     'cpu'  # ensure determinism for the device-dependent torch.Generator
    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     sd_pipe(
        2 * [prompt]    ,		image=2 * [low_res_image]    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)
    A__       =     output.images
    assert image.shape[0] == 2

    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		num_images_per_prompt=2    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)
    A__       =     output.images
    assert image.shape[0] == 2






 @unittest.skipIf(torch_device != 'cuda'    ,		'This test requires a GPU'    )
 def 						_UpperCamelCase (self       :  List[Any]    )       ->   Union[str, Any]:






    """simple docstring"""


    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # put models in fp16, except vae as it overflows in fp16
    A__       =     unet.half()
    A__       =     text_encoder.half()

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     torch.manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		num_inference_steps=2    ,		output_type='np'    ,		).images

    A__       =     low_res_image.size[0] * 4
    assert image.shape == (1, expected_height_width, expected_height_width, 3)




@slow
@require_torch_gpu
class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Dict:






    """simple docstring"""


    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()






 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )
    A__       =     load_numpy(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
        '/upsampled_cat.npy'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(a    )
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		output_type='np'    ,		)
    A__       =     output.images[0]

    assert image.shape == (5_12, 5_12, 3)
    assert np.abs(expected_image - image    ).max() < 1e-3






 def 						_UpperCamelCase (self       :  Any    )       ->   str:






    """simple docstring"""


    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )
    A__       =     load_numpy(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
        '/upsampled_cat_fp16.npy'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(
        a    ,		torch_dtype=torch.floataa    ,		)
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		output_type='np'    ,		)
    A__       =     output.images[0]

    assert image.shape == (5_12, 5_12, 3)
    assert np.abs(expected_image - image    ).max() < 5e-1






 def 						_UpperCamelCase (self       :  Dict    )       ->   List[str]:






    """simple docstring"""


    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.reset_peak_memory_stats()

    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(
        a    ,		torch_dtype=torch.floataa    ,		)
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing(1    )
    pipe.enable_sequential_cpu_offload()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		num_inference_steps=5    ,		output_type='np'    ,		)

    A__       =     torch.cuda.max_memory_allocated()
    # make sure that less than 2.9 GB is allocated
    assert mem_bytes < 2.9 * 10**9


