





from manim import *

class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





     """simple docstring"""


     def 					__magic_name__ ( self	):
          lowerCamelCase__       =       Rectangle(height=0.5	,  width=0.5	)
          lowerCamelCase__       =       Rectangle(height=0.25	,  width=0.25	)
          lowerCamelCase__       =       Rectangle(height=0.46	,  width=0.46	).set_stroke(width=0	)

          lowerCamelCase__       =       [mem.copy() for i in range(6	)]
          lowerCamelCase__       =       [mem.copy() for i in range(6	)]
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       VGroup(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       Text("CPU"	,  font_size=24	)
          lowerCamelCase__       =       Group(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0.5	,  aligned_edge=_lowerCAmelCase	)
          cpu.move_to([-2.5, -0.5, 0]	)
          self.add(_lowerCAmelCase	)

          lowerCamelCase__       =       [mem.copy() for i in range(4	)]
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       Text("GPU"	,  font_size=24	)
          lowerCamelCase__       =       Group(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0.5	,  aligned_edge=_lowerCAmelCase	)
          gpu.move_to([-1, -1, 0]	)
          self.add(_lowerCAmelCase	)

          lowerCamelCase__       =       [mem.copy() for i in range(6	)]
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)

          lowerCamelCase__       =       Text("Model"	,  font_size=24	)
          lowerCamelCase__       =       Group(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0.5	,  aligned_edge=_lowerCAmelCase	)
          model.move_to([3, -1.0, 0]	)
          self.add(_lowerCAmelCase	)

          lowerCamelCase__       =       []
          lowerCamelCase__       =       []
          lowerCamelCase__       =       []

          for i, rect in enumerate(_lowerCAmelCase	):
               rect.set_stroke(_lowerCAmelCase	)

               lowerCamelCase__       =       Rectangle(height=0.46 / 4	,  width=0.46 / 3	).set_stroke(width=0.0	).set_fill(_lowerCAmelCase	,  opacity=0.7	)

               if i == 0:
                    cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT	)	,  buff=0.02	,  direction=_lowerCAmelCase	)
                    cpu_target.set_x(cpu_target.get_x() + 0.1	)
               elif i == 3:
                    cpu_target.next_to(model_cpu_arr[0]	,  direction=_lowerCAmelCase	,  buff=0.0	)
               else:
                    cpu_target.next_to(model_cpu_arr[i - 1]	,  direction=_lowerCAmelCase	,  buff=0.0	)
               self.add(_lowerCAmelCase	)
               model_cpu_arr.append(_lowerCAmelCase	)

          self.add(*_lowerCAmelCase	,  *_lowerCAmelCase	,  *_lowerCAmelCase	)

          lowerCamelCase__       =       [mem.copy() for i in range(6	)]
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)

          lowerCamelCase__       =       Text("Loaded Checkpoint"	,  font_size=24	)
          lowerCamelCase__       =       Group(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0.5	,  aligned_edge=_lowerCAmelCase	)
          checkpoint.move_to([3, 0.5, 0]	)

          self.add(_lowerCAmelCase	)

          lowerCamelCase__       =       []
          lowerCamelCase__       =       []

          for i, rect in enumerate(_lowerCAmelCase	):
               lowerCamelCase__       =       fill.copy().set_fill(_lowerCAmelCase	,  opacity=0.7	)
               target.move_to(_lowerCAmelCase	)
               ckpt_arr.append(_lowerCAmelCase	)

               lowerCamelCase__       =       target.copy()
               if i < 5:
                    cpu_target.move_to(cpu_left_col_base[i + 1]	)
               else:
                    cpu_target.move_to(cpu_right_col_base[i - 5]	)
               ckpt_cpu_arr.append(_lowerCAmelCase	)
          self.add(*_lowerCAmelCase	,  *_lowerCAmelCase	)

          lowerCamelCase__       =       Square(side_length=2.2	)
          key.move_to([-5, 2, 0]	)

          lowerCamelCase__       =       MarkupText(
              F"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"	,  font_size=18	,  )

          key_text.move_to([-5, 2.4, 0]	)

          self.add(_lowerCAmelCase	,  _lowerCAmelCase	)

          lowerCamelCase__       =       MarkupText(
              F"<span fgcolor='{BLUE}'>●</span> Checkpoint"	,  font_size=18	,  )

          blue_text.next_to(_lowerCAmelCase	,  DOWN * 2.4	,  aligned_edge=key_text.get_left()	)
          self.add(_lowerCAmelCase	)

          lowerCamelCase__       =       MarkupText(
              F"Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."	,  font_size=24	,  )
          step_a.move_to([2, 2, 0]	)

          lowerCamelCase__       =       [meta_mem.copy() for i in range(6	)]
          lowerCamelCase__       =       [meta_mem.copy() for i in range(6	)]
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       VGroup(*_lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       VGroup(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0	)
          lowerCamelCase__       =       Text("Disk"	,  font_size=24	)
          lowerCamelCase__       =       Group(_lowerCAmelCase	,  _lowerCAmelCase	).arrange(_lowerCAmelCase	,  buff=0.5	,  aligned_edge=_lowerCAmelCase	)
          disk.move_to([-4.0, -1.25, 0]	)
          self.play(Write(_lowerCAmelCase	,  run_time=3	)	,  Write(_lowerCAmelCase	,  run_time=1	)	,  Create(_lowerCAmelCase	,  run_time=1	)	)

          lowerCamelCase__       =       []
          for i, rect in enumerate(_lowerCAmelCase	):
               lowerCamelCase__       =       rect.copy()
               target.generate_target()
               target.target.move_to(disk_left_col_base[i]	).scale(0.5	)
               animations.append(MoveToTarget(_lowerCAmelCase	,  run_time=1.5	)	)
          self.play(*_lowerCAmelCase	)

          self.play(FadeOut(_lowerCAmelCase	)	)

          lowerCamelCase__       =       MarkupText(F"Then, the checkpoint is removed from memory\nthrough garbage collection."	,  font_size=24	)
          step_a.move_to([2, 2, 0]	)

          self.play(Write(_lowerCAmelCase	,  run_time=3	)	)

          self.play(
              FadeOut(_lowerCAmelCase	,  _lowerCAmelCase	,  *_lowerCAmelCase	,  *_lowerCAmelCase	)	,  )

          self.wait()








import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_torch_available, is_vision_available


if is_torch_available():
						import torch


if is_vision_available():
						from PIL import Image

						from transformers import MgpstrProcessor, ViTImageProcessor

@require_torch
@require_vision
class SCREAMING_SNAKE_CASE_ (					unittest.TestCase   ):





					"""simple docstring"""


					A__				 =    ViTImageProcessor if is_vision_available() else None





					@property
					def 					__magic_name__ ( self	):
										return self.image_processor_tester.prepare_image_processor_dict()





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       (3, 32, 128)
										lowerCamelCase__       =       tempfile.mkdtemp()

										# fmt: off
										lowerCamelCase__       =       ["[GO]", "[s]", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
										# fmt: on
										lowerCamelCase__       =       dict(zip(_lowerCAmelCase	,  range(len(_lowerCAmelCase	)	)	)	)

										lowerCamelCase__       =       os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["vocab_file"]	)
										with open(self.vocab_file	,  "w"	,  encoding="utf-8"	) as fp:
															fp.write(json.dumps(_lowerCAmelCase	) + "\n"	)

										lowerCamelCase__       =       {
										    "do_normalize": False,
										    "do_resize": True,
										    "image_processor_type": "ViTImageProcessor",
										    "resample": 3,
										    "size": {"height": 32, "width": 128},
										}
										lowerCamelCase__       =       os.path.join(self.tmpdirname	,  _lowerCAmelCase	)
										with open(self.image_processor_file	,  "w"	,  encoding="utf-8"	) as fp:
															json.dump(_lowerCAmelCase	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	,  **_lowerCAmelCase	):
										return MgpstrTokenizer.from_pretrained(self.tmpdirname	,  **_lowerCAmelCase	)





					def 					__magic_name__ ( self	,  **_lowerCAmelCase	):
										return ViTImageProcessor.from_pretrained(self.tmpdirname	,  **_lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										shutil.rmtree(self.tmpdirname	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       np.random.randint(255	,  size=(3, 30, 400)	,  dtype=np.uinta	)

										lowerCamelCase__       =       Image.fromarray(np.moveaxis(_lowerCAmelCase	,  0	,  -1	)	)

										return image_input





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_tokenizer()
										lowerCamelCase__       =       self.get_image_processor()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)
										processor.save_pretrained(self.tmpdirname	)
										lowerCamelCase__       =       MgpstrProcessor.from_pretrained(self.tmpdirname	,  use_fast=_lowerCAmelCase	)

										self.assertEqual(processor.char_tokenizer.get_vocab()	,  tokenizer.get_vocab()	)
										self.assertIsInstance(processor.char_tokenizer	,  _lowerCAmelCase	)

										self.assertEqual(processor.image_processor.to_json_string()	,  image_processor.to_json_string()	)
										self.assertIsInstance(processor.image_processor	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_tokenizer()
										lowerCamelCase__       =       self.get_image_processor()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)
										processor.save_pretrained(self.tmpdirname	)

										lowerCamelCase__       =       self.get_tokenizer(bos_token="(BOS)"	,  eos_token="(EOS)"	)
										lowerCamelCase__       =       self.get_image_processor(do_normalize=_lowerCAmelCase	,  padding_value=1.0	)

										lowerCamelCase__       =       MgpstrProcessor.from_pretrained(
										    self.tmpdirname	,  bos_token="(BOS)"	,  eos_token="(EOS)"	,  do_normalize=_lowerCAmelCase	,  padding_value=1.0	)

										self.assertEqual(processor.char_tokenizer.get_vocab()	,  tokenizer_add_kwargs.get_vocab()	)
										self.assertIsInstance(processor.char_tokenizer	,  _lowerCAmelCase	)

										self.assertEqual(processor.image_processor.to_json_string()	,  image_processor_add_kwargs.to_json_string()	)
										self.assertIsInstance(processor.image_processor	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_image_processor()
										lowerCamelCase__       =       self.get_tokenizer()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)

										lowerCamelCase__       =       self.prepare_image_inputs()

										lowerCamelCase__       =       image_processor(_lowerCAmelCase	,  return_tensors="np"	)
										lowerCamelCase__       =       processor(images=_lowerCAmelCase	,  return_tensors="np"	)

										for key in input_image_proc.keys():
															self.assertAlmostEqual(input_image_proc[key].sum()	,  input_processor[key].sum()	,  delta=1E-2	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_image_processor()
										lowerCamelCase__       =       self.get_tokenizer()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)

										lowerCamelCase__       =       "test"

										lowerCamelCase__       =       processor(text=_lowerCAmelCase	)

										lowerCamelCase__       =       tokenizer(_lowerCAmelCase	)
										for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]	,  encoded_processor[key]	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_image_processor()
										lowerCamelCase__       =       self.get_tokenizer()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)

										lowerCamelCase__       =       "test"
										lowerCamelCase__       =       self.prepare_image_inputs()

										lowerCamelCase__       =       processor(text=_lowerCAmelCase	,  images=_lowerCAmelCase	)

										self.assertListEqual(list(inputs.keys()	)	,  ["pixel_values", "labels"]	)

										# test if it raises when no input is passed
										with pytest.raises(_lowerCAmelCase	):
															processor()





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_image_processor()
										lowerCamelCase__       =       self.get_tokenizer()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)

										lowerCamelCase__       =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9], [3, 4, 3, 1, 1, 8, 9]]

										lowerCamelCase__       =       processor.char_decode(_lowerCAmelCase	)
										lowerCamelCase__       =       tokenizer.batch_decode(_lowerCAmelCase	)
										lowerCamelCase__       =       [seq.replace(" "	,  ""	) for seq in decoded_tok]

										self.assertListEqual(_lowerCAmelCase	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_image_processor()
										lowerCamelCase__       =       self.get_tokenizer()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)

										lowerCamelCase__       =       None
										lowerCamelCase__       =       self.prepare_image_inputs()

										lowerCamelCase__       =       processor(text=_lowerCAmelCase	,  images=_lowerCAmelCase	)

										self.assertListEqual(list(inputs.keys()	)	,  processor.model_input_names	)






					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.get_image_processor()
										lowerCamelCase__       =       self.get_tokenizer()

										lowerCamelCase__       =       MgpstrProcessor(tokenizer=_lowerCAmelCase	,  image_processor=_lowerCAmelCase	)

										lowerCamelCase__       =       torch.randn(1	,  27	,  38	)
										lowerCamelCase__       =       torch.randn(1	,  27	,  5_0257	)
										lowerCamelCase__       =       torch.randn(1	,  27	,  3_0522	)

										lowerCamelCase__       =       processor.batch_decode([char_input, bpe_input, wp_input]	)

										self.assertListEqual(list(results.keys()	)	,  ["generated_text", "scores", "char_preds", "bpe_preds", "wp_preds"]	)

