




'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_snake_case   :   Optional[Any]     							=	logging.get_logger(__name__)

_snake_case   :   Tuple     							=	{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case   :   List[Any]     							=	{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case   :   Any     							=	{
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


_snake_case   :   Any     							=	{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}

class     A					(	_a					):
							lowercase_           =       VOCAB_FILES_NAMES
							lowercase_           =       PRETRAINED_VOCAB_FILES_MAP
							lowercase_           =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_           =       PRETRAINED_INIT_CONFIGURATION
							lowercase_           =       ['input_ids', 'attention_mask']
							lowercase_           =       DistilBertTokenizer







							def __init__(						self :						int							,    lowerCAmelCase_ :						List[str]=None							,    lowerCAmelCase_ :						Dict=None							,    lowerCAmelCase_ :						Optional[Any]=True							,    lowerCAmelCase_ :						str="[UNK]"							,    lowerCAmelCase_ :						List[Any]="[SEP]"							,    lowerCAmelCase_ :						int="[PAD]"							,    lowerCAmelCase_ :						Optional[Any]="[CLS]"							,    lowerCAmelCase_ :						Optional[Any]="[MASK]"							,    lowerCAmelCase_ :						int=True							,    lowerCAmelCase_ :						List[str]=None							,    **lowerCAmelCase_ :						Tuple							,    )    ->							Optional[Any]:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,    tokenizer_file=lowerCAmelCase_							,    do_lower_case=lowerCAmelCase_							,    unk_token=lowerCAmelCase_							,    sep_token=lowerCAmelCase_							,    pad_token=lowerCAmelCase_							,    cls_token=lowerCAmelCase_							,    mask_token=lowerCAmelCase_							,    tokenize_chinese_chars=lowerCAmelCase_							,    strip_accents=lowerCAmelCase_							,    **lowerCAmelCase_							,    )

														_a			  =       json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
														if (
														    normalizer_state.get('''lowercase'''							,    lowerCAmelCase_      ) != do_lower_case
														    or normalizer_state.get('''strip_accents'''							,    lowerCAmelCase_      ) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''							,    lowerCAmelCase_      ) != tokenize_chinese_chars
														):
																					_a			  =       getattr(lowerCAmelCase_							,    normalizer_state.pop('''type'''      )      )
																					_a			  =       do_lower_case
																					_a			  =       strip_accents
																					_a			  =       tokenize_chinese_chars
																					_a			  =       normalizer_class(**lowerCAmelCase_      )

														_a			  =       do_lower_case







							def      __lowerCAmelCase    (						self :						List[str]							,    lowerCAmelCase_ :						Optional[int]							,    lowerCAmelCase_ :						Dict=None      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def      __lowerCAmelCase    (						self :						Tuple							,    lowerCAmelCase_ :						List[int]							,    lowerCAmelCase_ :						Optional[List[int]] = None      )    ->							List[int]:


														"""simple docstring"""


														_a			  =       [self.sep_token_id]
														_a			  =       [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep      ) * [0]
														return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







							def      __lowerCAmelCase    (						self :						Tuple							,    lowerCAmelCase_ :						str							,    lowerCAmelCase_ :						Optional[str] = None      )    ->							Tuple[str]:


														"""simple docstring"""


														_a			  =       self._tokenizer.model.save(lowerCAmelCase_							,    name=lowerCAmelCase_      )
														return tuple(lowerCAmelCase_      )







'''simple docstring'''


from manim import *

class     A					(	_a					):







							def      __lowerCAmelCase    (						self :						Tuple      )    ->							Dict:


														"""simple docstring"""


														_a			  =       Rectangle(height=0.5							,    width=0.5      )
														_a			  =       Rectangle(height=0.4_6							,    width=0.4_6      ).set_stroke(width=0      )

														_a			  =       [mem.copy() for i in range(6      )]
														_a			  =       [mem.copy() for i in range(6      )]
														_a			  =       VGroup(*lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0      )
														_a			  =       VGroup(*lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0      )
														_a			  =       VGroup(lowerCAmelCase_							,    lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0      )
														_a			  =       Text('''CPU'''							,    font_size=24      )
														_a			  =       Group(lowerCAmelCase_							,    lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0.5							,    aligned_edge=lowerCAmelCase_      )
														cpu.move_to([-2.5, -0.5, 0]      )
														self.add(lowerCAmelCase_      )

														_a			  =       [mem.copy() for i in range(4      )]
														_a			  =       VGroup(*lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0      )
														_a			  =       Text('''GPU'''							,    font_size=24      )
														_a			  =       Group(lowerCAmelCase_							,    lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0.5							,    aligned_edge=lowerCAmelCase_      )
														gpu.move_to([-1, -1, 0]      )
														self.add(lowerCAmelCase_      )

														_a			  =       [mem.copy() for i in range(6      )]
														_a			  =       VGroup(*lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0      )

														_a			  =       Text('''Model'''							,    font_size=24      )
														_a			  =       Group(lowerCAmelCase_							,    lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0.5							,    aligned_edge=lowerCAmelCase_      )
														model.move_to([3, -1.0, 0]      )
														self.add(lowerCAmelCase_      )

														_a			  =       []
														for i, rect in enumerate(lowerCAmelCase_      ):
																					rect.set_stroke(lowerCAmelCase_      )
																					# target = fill.copy().set_fill(YELLOW, opacity=0.7)
																					# target.move_to(rect)
																					# self.add(target)

																					_a			  =       Rectangle(height=0.4_6 / 4							,    width=0.4_6 / 3      ).set_stroke(width=0.0      ).set_fill(lowerCAmelCase_							,    opacity=0.7      )

																					if i == 0:
																												cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT      )							,    buff=0.0_2							,    direction=lowerCAmelCase_      )
																												cpu_target.set_x(cpu_target.get_x() + 0.1      )
																					elif i == 3:
																												cpu_target.next_to(cpu_targs[0]							,    direction=lowerCAmelCase_							,    buff=0.0      )
																					else:
																												cpu_target.next_to(cpu_targs[i - 1]							,    direction=lowerCAmelCase_							,    buff=0.0      )
																					self.add(lowerCAmelCase_      )
																					cpu_targs.append(lowerCAmelCase_      )

														_a			  =       [mem.copy() for i in range(6      )]
														_a			  =       VGroup(*lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    buff=0      )

														_a			  =       Text('''Loaded Checkpoint'''							,    font_size=24      )
														_a			  =       Group(lowerCAmelCase_							,    lowerCAmelCase_      ).arrange(lowerCAmelCase_							,    aligned_edge=lowerCAmelCase_							,    buff=0.4      )
														checkpoint.move_to([3, 0.5, 0]      )

														_a			  =       Square(side_length=2.2      )
														key.move_to([-5, 2, 0]      )

														_a			  =       MarkupText(
														    F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'							,    font_size=18							,    )

														key_text.move_to([-5, 2.4, 0]      )

														self.add(lowerCAmelCase_							,    lowerCAmelCase_      )

														_a			  =       MarkupText(
														    F'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'							,    font_size=18							,    )

														blue_text.next_to(lowerCAmelCase_							,    DOWN * 2.4							,    aligned_edge=key_text.get_left()      )

														_a			  =       MarkupText(
														    F'Next, a <i><span fgcolor="{BLUE}">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor="{BLUE}">single shard</span>.'							,    font_size=24							,    )
														step_a.move_to([2, 2, 0]      )
														self.play(Write(lowerCAmelCase_      )							,    Write(lowerCAmelCase_      )      )

														self.play(Write(lowerCAmelCase_							,    run_time=1      )							,    Create(lowerCAmelCase_							,    run_time=1      )      )

														_a			  =       []
														_a			  =       []
														for i, rect in enumerate(lowerCAmelCase_      ):
																					_a			  =       fill.copy().set_fill(lowerCAmelCase_							,    opacity=0.7      )
																					target.move_to(lowerCAmelCase_      )
																					first_animations.append(GrowFromCenter(lowerCAmelCase_							,    run_time=1      )      )

																					_a			  =       target.copy()
																					cpu_target.generate_target()
																					if i < 5:
																												cpu_target.target.move_to(cpu_left_col_base[i + 1]      )
																					else:
																												cpu_target.target.move_to(cpu_right_col_base[i - 5]      )
																					second_animations.append(MoveToTarget(lowerCAmelCase_							,    run_time=1.5      )      )

														self.play(*lowerCAmelCase_      )
														self.play(*lowerCAmelCase_      )
														self.wait()

