


from manim import *





class   __a      (   __UpperCamelCase				):
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[str]:




        '''simple docstring'''
        lowercase__:       Dict        =       Rectangle(height=0.5    ,			width=0.5			)
        lowercase__:       str        =       Rectangle(height=0.4_6    ,			width=0.4_6			).set_stroke(width=0			)

        lowercase__:       Optional[Any]        =       [mem.copy() for i in range(6			)]
        lowercase__:       str        =       [mem.copy() for i in range(6			)]
        lowercase__:       Dict        =       VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0			)
        lowercase__:       Optional[int]        =       VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0			)
        lowercase__:       Optional[Any]        =       VGroup(lowerCAmelCase__    ,			lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0			)
        lowercase__:       Optional[Any]        =       Text('CPU'    ,			font_size=24			)
        lowercase__:       List[str]        =       Group(lowerCAmelCase__    ,			lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0.5    ,			aligned_edge=lowerCAmelCase__			)
        cpu.move_to([-2.5, -0.5, 0]			)
        self.add(lowerCAmelCase__			)

        lowercase__:       Dict        =       [mem.copy() for i in range(1			)]
        lowercase__:       List[Any]        =       VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0			)
        lowercase__:       int        =       Text('GPU'    ,			font_size=24			)
        lowercase__:       Dict        =       Group(lowerCAmelCase__    ,			lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0.5    ,			aligned_edge=lowerCAmelCase__			)
        gpu.align_to(lowerCAmelCase__    ,			lowerCAmelCase__			)
        gpu.set_x(gpu.get_x() - 1			)

        self.add(lowerCAmelCase__			)

        lowercase__:       List[str]        =       [mem.copy() for i in range(6			)]
        lowercase__:       Dict        =       VGroup(*lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0			)

        lowercase__:       Union[str, Any]        =       Text('Model'    ,			font_size=24			)
        lowercase__:       Any        =       Group(lowerCAmelCase__    ,			lowerCAmelCase__			).arrange(lowerCAmelCase__    ,			buff=0.5    ,			aligned_edge=lowerCAmelCase__			)
        model.move_to([3, -1.0, 0]			)

        self.play(
            Create(lowerCAmelCase__    ,			run_time=1			)    ,			Create(lowerCAmelCase__    ,			run_time=1			)    ,			Create(lowerCAmelCase__    ,			run_time=1			)    ,			)

        lowercase__:       int        =       MarkupText(
            F'First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.'    ,			font_size=24    ,			)

        lowercase__:       List[Any]        =       Square(side_length=2.2			)
        key.move_to([-5, 2, 0]			)

        lowercase__:       Optional[int]        =       MarkupText(
            F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'    ,			font_size=18    ,			)

        key_text.move_to([-5, 2.4, 0]			)

        step_a.move_to([2, 2, 0]			)
        self.play(Write(lowerCAmelCase__    ,			run_time=2.5			)    ,			Write(lowerCAmelCase__			)    ,			Write(lowerCAmelCase__			)			)

        self.add(lowerCAmelCase__			)

        lowercase__:       Dict        =       []
        lowercase__:       Optional[Any]        =       []
        lowercase__:       List[Any]        =       []
        for i, rect in enumerate(lowerCAmelCase__			):
               lowercase__:       Union[str, Any]        =       Rectangle(height=0.4_6    ,			width=0.4_6			).set_stroke(width=0.0			).set_fill(lowerCAmelCase__    ,			opacity=0.7			)
               cpu_target.move_to(lowerCAmelCase__			)
               cpu_target.generate_target()
               lowercase__:       List[Any]        =       0.4_6 / 4
               lowercase__:       Optional[int]        =       0.4_6 / 3

               if i == 0:
                      cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT			)    ,			buff=0.0_2    ,			direction=lowerCAmelCase__			)
                      cpu_target.target.set_x(cpu_target.target.get_x() + 0.1			)
               elif i == 3:
                      cpu_target.target.next_to(cpu_targs[0].target    ,			direction=lowerCAmelCase__    ,			buff=0.0			)
               else:
                      cpu_target.target.next_to(cpu_targs[i - 1].target    ,			direction=lowerCAmelCase__    ,			buff=0.0			)
               cpu_targs.append(lowerCAmelCase__			)

               first_animations.append(rect.animate(run_time=0.5			).set_stroke(lowerCAmelCase__			)			)
               second_animations.append(MoveToTarget(lowerCAmelCase__    ,			run_time=1.5			)			)

        self.play(*lowerCAmelCase__			)
        self.play(*lowerCAmelCase__			)

        self.wait()






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class   __a      (   __UpperCamelCase	,	unittest.TestCase				):
 __lowercase			:							Optional[int]       					=			CLIPTokenizer
 __lowercase			:							str       					=			CLIPTokenizerFast
 __lowercase			:							Tuple       					=			True
 __lowercase			:							str       					=			{}
 __lowercase			:							Dict       					=			False
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        super().setUp()

        # fmt: off
        lowercase__:       str        =       ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
        # fmt: on
        lowercase__:       List[str]        =       dict(zip(lowerCAmelCase__    ,			range(len(lowerCAmelCase__			)			)			)			)
        lowercase__:       int        =       ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
        lowercase__:       Optional[int]        =       {'unk_token': '<unk>'}

        lowercase__:       int        =       os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES['vocab_file']			)
        lowercase__:       Union[str, Any]        =       os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES['merges_file']			)
        with open(self.vocab_file    ,			'w'    ,			encoding='utf-8'			) as fp:
               fp.write(json.dumps(lowerCAmelCase__			) + '\n'			)
        with open(self.merges_file    ,			'w'    ,			encoding='utf-8'			) as fp:
               fp.write('\n'.join(lowerCAmelCase__			)			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			**lowerCAmelCase__			) ->    List[str]:




        '''simple docstring'''
        kwargs.update(self.special_tokens_map			)
        return CLIPTokenizer.from_pretrained(self.tmpdirname    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			**lowerCAmelCase__			) ->    Optional[int]:




        '''simple docstring'''
        kwargs.update(self.special_tokens_map			)
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    Union[str, Any]:




        '''simple docstring'''
        lowercase__:       Dict        =       'lower newer'
        lowercase__:       Dict        =       'lower newer'
        return input_text, output_text
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[Any]:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       CLIPTokenizer(self.vocab_file    ,			self.merges_file    ,			**self.special_tokens_map			)
        lowercase__:       Dict        =       'lower newer'
        lowercase__:       Union[str, Any]        =       ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
        lowercase__:       Any        =       tokenizer.tokenize(lowerCAmelCase__			)
        self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

        lowercase__:       Tuple        =       tokens + [tokenizer.unk_token]
        lowercase__:       Tuple        =       [10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)    ,			lowerCAmelCase__			)
 @require_ftfy
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Tuple:




        '''simple docstring'''
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                      lowercase__:       List[str]        =       self.tokenizer_class.from_pretrained(lowerCAmelCase__    ,			**lowerCAmelCase__			)
                      lowercase__:       Optional[int]        =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__    ,			**lowerCAmelCase__			)

                      lowercase__:       Optional[int]        =       'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
                      lowercase__:       Optional[Any]        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                      lowercase__:       Dict        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                      self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

                      # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                      # with Tilde) encoded in 2 different ways
                      lowercase__:       Dict        =       'xa\u0303y' + ' ' + 'x\xe3y'
                      lowercase__:       Tuple        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                      lowercase__:       int        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                      self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

                      # Test that the tokenization is identical on unicode of space type
                      lowercase__:       str        =       [
                          '\u0009',  # (horizontal tab, '\t')
                          '\u000B',  # (vertical tab)
                          '\u000C',  # (form feed)
                          '\u0020',  # (space, ' ')
                          '\u200E',  # (left-to-right mark):w
                          '\u200F',  # (right-to-left mark)
                      ]
                      for unicode_seq in spaces_unicodes:
                             lowercase__:       Optional[Any]        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                             lowercase__:       Tuple        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                             self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

                      # Test that the tokenization is identical on unicode of line break type
                      lowercase__:       str        =       [
                          '\u000A',  # (line feed, '\n')
                          '\r\n',  # (carriage return and line feed, '\r\n')
                          '\u000D',  # (carriage return, '\r')
                          '\r',  # (carriage return, '\r')
                          '\u000D',  # (carriage return, '\r')
                          '\u2028',  # (line separator)
                          '\u2029',  # (paragraph separator)
                          # "\u0085", # (next line)
                      ]

                      # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                      # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                      # space (and thus into an empty list).

                      for unicode_seq in line_break_unicodes:
                             lowercase__:       Optional[int]        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                             lowercase__:       Optional[int]        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                             self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Optional[int]:




        '''simple docstring'''
        # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                      lowercase__:       Optional[int]        =       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                      lowercase__:       Optional[int]        =       F'{text_of_1_token} {text_of_1_token}'

                      lowercase__:       int        =       self.rust_tokenizer_class.from_pretrained(
                          lowerCAmelCase__    ,			use_fast=lowerCAmelCase__    ,			)
                      lowercase__:       Dict        =       tokenizer_r(lowerCAmelCase__    ,			return_offsets_mapping=lowerCAmelCase__    ,			add_special_tokens=lowerCAmelCase__			)
                      self.assertEqual(encoding.offset_mapping[0]    ,			(0, len(lowerCAmelCase__			))			)
                      self.assertEqual(
                          encoding.offset_mapping[1]    ,			(len(lowerCAmelCase__			) + 1, len(lowerCAmelCase__			) + 1 + len(lowerCAmelCase__			))    ,			)

                      lowercase__:       Any        =       F' {text}'

                      lowercase__:       Union[str, Any]        =       self.rust_tokenizer_class.from_pretrained(
                          lowerCAmelCase__    ,			use_fast=lowerCAmelCase__    ,			)
                      lowercase__:       int        =       tokenizer_r(lowerCAmelCase__    ,			return_offsets_mapping=lowerCAmelCase__    ,			add_special_tokens=lowerCAmelCase__			)
                      self.assertEqual(encoding.offset_mapping[0]    ,			(1, 1 + len(lowerCAmelCase__			))			)
                      self.assertEqual(
                          encoding.offset_mapping[1]    ,			(1 + len(lowerCAmelCase__			) + 1, 1 + len(lowerCAmelCase__			) + 1 + len(lowerCAmelCase__			))    ,			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[str]:




        '''simple docstring'''
        # Test related to the breaking change introduced in transformers v4.17.0
        # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
        with self.assertRaises(lowerCAmelCase__			) as context:
               self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'			)

        self.assertTrue(
            context.exception.args[0].startswith(
                'The `backend_tokenizer` provided does not match the expected format.'			)			)
 @require_ftfy
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[Any]:




        '''simple docstring'''
        super().test_tokenization_python_rust_equals()
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Tuple:




        '''simple docstring'''
        # CLIP always lower cases letters
        pass


