"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation

__lowerCamelCase											=  logging.get_logger(__name__)


__lowerCamelCase											=  {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

__lowerCamelCase											=  {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

__lowerCamelCase											=  {"facebook/blenderbot-3B": 1_28}






@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   UpperCAmelCase					(	):






							"""simple docstring"""





							A__	     =							(
							    list(range(ord('!'				)       ,    ord('~'				) + 1				)				) + list(range(ord('¡'				)       ,    ord('¬'				) + 1				)				) + list(range(ord('®'				)       ,    ord('ÿ'				) + 1				)				)
							)
							A__	     =							bs[:]
							A__	     =							0
							for b in range(2**8				):
														if b not in bs:
																					bs.append(UpperCamelCase__				)
																					cs.append(2**8 + n				)
																					n += 1
							A__	     =							[chr(UpperCamelCase__				) for n in cs]
							return dict(zip(UpperCamelCase__       ,    UpperCamelCase__				)				)






def   UpperCAmelCase					(	UpperCamelCase__				):






							"""simple docstring"""





							A__	     =							set()
							A__	     =							word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)				)
														A__	     =							char
							return pairs


class     UpperCamelCase__(			__A ):
						lowerCAmelCase__  :						Optional[int]													=		VOCAB_FILES_NAMES
						lowerCAmelCase__  :						Dict													=		PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase__  :						int													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase__  :						Union[str, Any]													=		['input_ids', 'attention_mask']





						def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase="replace"		,__UpperCAmelCase="<s>"		,__UpperCAmelCase="</s>"		,__UpperCAmelCase="</s>"		,__UpperCAmelCase="<s>"		,__UpperCAmelCase="<unk>"		,__UpperCAmelCase="<pad>"		,__UpperCAmelCase="<mask>"		,__UpperCAmelCase=False		,**__UpperCAmelCase		,)					->				Optional[Any]:
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else bos_token
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else eos_token
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else sep_token
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else cls_token
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else unk_token
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else pad_token

													# Mask token behave like a normal word, i.e. include the space before it
													A__	     =							AddedToken(__UpperCAmelCase		,lstrip=__UpperCAmelCase		,rstrip=__UpperCAmelCase     ) if isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) else mask_token

													super().__init__(
													    errors=__UpperCAmelCase		,bos_token=__UpperCAmelCase		,eos_token=__UpperCAmelCase		,unk_token=__UpperCAmelCase		,sep_token=__UpperCAmelCase		,cls_token=__UpperCAmelCase		,pad_token=__UpperCAmelCase		,mask_token=__UpperCAmelCase		,add_prefix_space=__UpperCAmelCase		,**__UpperCAmelCase		,)

													with open(__UpperCAmelCase		,encoding='utf-8'     ) as vocab_handle:
																				A__	     =							json.load(__UpperCAmelCase     )
													A__	     =							{v: k for k, v in self.encoder.items()}
													A__	     =							errors  # how to handle errors in decoding
													A__	     =							bytes_to_unicode()
													A__	     =							{v: k for k, v in self.byte_encoder.items()}
													with open(__UpperCAmelCase		,encoding='utf-8'     ) as merges_handle:
																				A__	     =							merges_handle.read().split('\n'     )[1:-1]
													A__	     =							[tuple(merge.split()     ) for merge in bpe_merges]
													A__	     =							dict(zip(__UpperCAmelCase		,range(len(__UpperCAmelCase     )     )     )     )
													A__	     =							{}
													A__	     =							add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													A__	     =							re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )





						@property
						# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
						def 				snake_case__	(       self     )					->				Optional[int]:
													return len(self.encoder     )





						def 				snake_case__	(       self     )					->				Union[str, Any]:
													return dict(self.encoder		,**self.added_tokens_encoder     )





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				List[Any]:
													if token in self.cache:
																				return self.cache[token]
													A__	     =							tuple(__UpperCAmelCase     )
													A__	     =							get_pairs(__UpperCAmelCase     )

													if not pairs:
																				return token

													while True:
																				A__	     =							min(__UpperCAmelCase		,key=lambda __UpperCAmelCase     : self.bpe_ranks.get(__UpperCAmelCase		,float('inf'     )     )     )
																				if bigram not in self.bpe_ranks:
																											break
																				A__     ,  A__	     =							bigram
																				A__	     =							[]
																				A__	     =							0
																				while i < len(__UpperCAmelCase     ):
																											try:
																																		A__	     =							word.index(__UpperCAmelCase		,__UpperCAmelCase     )
																											except ValueError:
																																		new_word.extend(word[i:]     )
																																		break
																											else:
																																		new_word.extend(word[i:j]     )
																																		A__	     =							j

																											if word[i] == first and i < len(__UpperCAmelCase     ) - 1 and word[i + 1] == second:
																																		new_word.append(first + second     )
																																		i += 2
																											else:
																																		new_word.append(word[i]     )
																																		i += 1
																				A__	     =							tuple(__UpperCAmelCase     )
																				A__	     =							new_word
																				if len(__UpperCAmelCase     ) == 1:
																											break
																				else:
																											A__	     =							get_pairs(__UpperCAmelCase     )
													A__	     =							' '.join(__UpperCAmelCase     )
													A__	     =							word
													return word





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Optional[Any]:
													A__	     =							[]
													for token in re.findall(self.pat		,__UpperCAmelCase     ):
																				A__	     =							''.join(
																				    self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																				bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCAmelCase     ).split(' '     )     )
													return bpe_tokens





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Union[str, Any]:
													return self.encoder.get(__UpperCAmelCase		,self.encoder.get(self.unk_token     )     )





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Union[str, Any]:
													return self.decoder.get(__UpperCAmelCase     )





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Any:
													A__	     =							''.join(__UpperCAmelCase     )
													A__	     =							bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8'		,errors=self.errors     )
													return text





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None     )					->				Tuple[str]:
													if not os.path.isdir(__UpperCAmelCase     ):
																				logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
																				return
													A__	     =							os.path.join(
													    __UpperCAmelCase		,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
													A__	     =							os.path.join(
													    __UpperCAmelCase		,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

													with open(__UpperCAmelCase		,'w'		,encoding='utf-8'     ) as f:
																				f.write(json.dumps(self.encoder		,indent=2		,sort_keys=__UpperCAmelCase		,ensure_ascii=__UpperCAmelCase     ) + '\n'     )

													A__	     =							0
													with open(__UpperCAmelCase		,'w'		,encoding='utf-8'     ) as writer:
																				writer.write('#version: 0.2\n'     )
																				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,key=lambda __UpperCAmelCase     : kv[1]     ):
																											if index != token_index:
																																		logger.warning(
																																		    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																																		    ' Please check that the tokenizer is not corrupted!'     )
																																		A__	     =							token_index
																											writer.write(' '.join(__UpperCAmelCase     ) + '\n'     )
																											index += 1

													return vocab_file, merge_file





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None		,__UpperCAmelCase = False     )					->				List[int]:
													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=__UpperCAmelCase		,token_ids_a=__UpperCAmelCase		,already_has_special_tokens=__UpperCAmelCase     )

													if token_ids_a is None:
																				return [1] + ([0] * len(__UpperCAmelCase     )) + [1]
													return [1] + ([0] * len(__UpperCAmelCase     )) + [1, 1] + ([0] * len(__UpperCAmelCase     )) + [1]





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None     )					->				List[int]:
													A__	     =							[self.sep_token_id]
													A__	     =							[self.cls_token_id]

													if token_ids_a is None:
																				return len(cls + token_ids_a + sep     ) * [0]
													return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase=False		,**__UpperCAmelCase     )					->				Optional[Any]:
													A__	     =							kwargs.pop('add_prefix_space'		,self.add_prefix_space     )
													if (is_split_into_words or add_prefix_space) and (len(__UpperCAmelCase     ) > 0 and not text[0].isspace()):
																				A__	     =							' ' + text
													return (text, kwargs)





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None     )					->				Union[str, Any]:
													return token_ids_a + [self.eos_token_id]





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				List[int]:
													A__	     =							[]
													for is_user, text in conversation.iter_texts():
																				if is_user:
																											# We need to space prefix as it's being done within blenderbot
																											inputs.append(' ' + text     )
																				else:
																											# Generated responses should contain them already.
																											inputs.append(__UpperCAmelCase     )

													A__	     =							'  '.join(__UpperCAmelCase     )
													A__	     =							self.encode(__UpperCAmelCase     )
													if len(__UpperCAmelCase     ) > self.model_max_length:
																				A__	     =							input_ids[-self.model_max_length :]
																				logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''     )
													return input_ids






"""simple docstring"""




from manim import *
class     UpperCamelCase__(			__A ):





						def 				snake_case__	(       self     )					->				List[str]:
													A__	     =							Rectangle(height=0.5		,width=0.5     )
													A__	     =							Rectangle(height=0.4_6		,width=0.4_6     ).set_stroke(width=0     )

													A__	     =							Rectangle(height=0.2_5		,width=0.2_5     )

													A__	     =							[mem.copy() for i in range(6     )]
													A__	     =							[mem.copy() for i in range(6     )]
													A__	     =							VGroup(*__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							VGroup(*__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							VGroup(__UpperCAmelCase		,__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							Text('CPU'		,font_size=24     )
													A__	     =							Group(__UpperCAmelCase		,__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0.5		,aligned_edge=__UpperCAmelCase     )
													cpu.move_to([-2.5, -0.5, 0]     )
													self.add(__UpperCAmelCase     )

													A__	     =							[mem.copy() for i in range(4     )]
													A__	     =							VGroup(*__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							Text('GPU'		,font_size=24     )
													A__	     =							Group(__UpperCAmelCase		,__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0.5		,aligned_edge=__UpperCAmelCase     )
													gpu.move_to([-1, -1, 0]     )
													self.add(__UpperCAmelCase     )

													A__	     =							[mem.copy() for i in range(6     )]
													A__	     =							VGroup(*__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )

													A__	     =							Text('Model'		,font_size=24     )
													A__	     =							Group(__UpperCAmelCase		,__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0.5		,aligned_edge=__UpperCAmelCase     )
													model.move_to([3, -1.0, 0]     )
													self.add(__UpperCAmelCase     )

													A__	     =							[]
													A__	     =							[]

													for i, rect in enumerate(__UpperCAmelCase     ):
																				A__	     =							fill.copy().set_fill(__UpperCAmelCase		,opacity=0.8     )
																				target.move_to(__UpperCAmelCase     )
																				model_arr.append(__UpperCAmelCase     )

																				A__	     =							Rectangle(height=0.4_6		,width=0.4_6     ).set_stroke(width=0.0     ).set_fill(__UpperCAmelCase		,opacity=0.8     )
																				cpu_target.move_to(cpu_left_col_base[i]     )
																				model_cpu_arr.append(__UpperCAmelCase     )

													self.add(*__UpperCAmelCase		,*__UpperCAmelCase     )

													A__	     =							[meta_mem.copy() for i in range(6     )]
													A__	     =							[meta_mem.copy() for i in range(6     )]
													A__	     =							VGroup(*__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							VGroup(*__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							VGroup(__UpperCAmelCase		,__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0     )
													A__	     =							Text('Disk'		,font_size=24     )
													A__	     =							Group(__UpperCAmelCase		,__UpperCAmelCase     ).arrange(__UpperCAmelCase		,buff=0.5		,aligned_edge=__UpperCAmelCase     )
													disk.move_to([-4, -1.2_5, 0]     )
													self.add(__UpperCAmelCase		,__UpperCAmelCase     )

													A__	     =							Square(side_length=2.2     )
													key.move_to([-5, 2, 0]     )

													A__	     =							MarkupText(
													    f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''		,font_size=18		,)

													key_text.move_to([-5, 2.4, 0]     )

													self.add(__UpperCAmelCase		,__UpperCAmelCase     )

													A__	     =							MarkupText(
													    f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''		,font_size=18		,)

													blue_text.next_to(__UpperCAmelCase		,DOWN * 2.4		,aligned_edge=key_text.get_left()     )
													self.add(__UpperCAmelCase     )

													A__	     =							MarkupText(
													    f'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.'''		,font_size=24		,)
													step_a.move_to([2, 2, 0]     )

													self.play(Write(__UpperCAmelCase     )     )

													A__	     =							Square(0.3     )
													input.set_fill(__UpperCAmelCase		,opacity=1.0     )
													input.set_stroke(width=0.0     )
													input.next_to(model_base[0]		,__UpperCAmelCase		,buff=0.5     )

													self.play(Write(__UpperCAmelCase     )     )

													input.generate_target()
													input.target.next_to(model_arr[0]		,direction=__UpperCAmelCase		,buff=0.0_2     )
													self.play(MoveToTarget(__UpperCAmelCase     )     )

													self.play(FadeOut(__UpperCAmelCase     )     )

													A__	     =							Arrow(start=__UpperCAmelCase		,end=__UpperCAmelCase		,color=__UpperCAmelCase		,buff=0.5     )
													a.next_to(model_arr[0].get_left()		,__UpperCAmelCase		,buff=0.2     )

													model_cpu_arr[0].generate_target()
													model_cpu_arr[0].target.move_to(gpu_rect[0]     )

													A__	     =							MarkupText(
													    f'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'''		,font_size=24		,)
													step_a.move_to([2, 2, 0]     )

													self.play(Write(__UpperCAmelCase		,run_time=3     )     )

													A__	     =							{'run_time': 1, 'fade_in': True, 'fade_out': True, 'buff': 0.0_2}

													self.play(
													    Write(__UpperCAmelCase     )		,Circumscribe(model_arr[0]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,Circumscribe(model_cpu_arr[0]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,Circumscribe(gpu_rect[0]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,)
													self.play(MoveToTarget(model_cpu_arr[0]     )     )

													A__	     =							a.copy()
													for i in range(6     ):
																				a_c.next_to(model_arr[i].get_right() + 0.0_2		,__UpperCAmelCase		,buff=0.2     )

																				input.generate_target()
																				input.target.move_to(model_arr[i].get_right() + 0.0_2     )

																				A__	     =							AnimationGroup(
																				    FadeOut(__UpperCAmelCase		,run_time=0.5     )		,MoveToTarget(__UpperCAmelCase		,run_time=0.5     )		,FadeIn(__UpperCAmelCase		,run_time=0.5     )		,lag_ratio=0.2     )

																				self.play(__UpperCAmelCase     )

																				model_cpu_arr[i].generate_target()
																				model_cpu_arr[i].target.move_to(cpu_left_col_base[i]     )

																				if i < 5:
																											model_cpu_arr[i + 1].generate_target()
																											model_cpu_arr[i + 1].target.move_to(gpu_rect[0]     )
																											if i >= 1:
																																		A__	     =							0.7

																											self.play(
																											    Circumscribe(model_arr[i]		,**__UpperCAmelCase     )		,Circumscribe(cpu_left_col_base[i]		,**__UpperCAmelCase     )		,Circumscribe(cpu_left_col_base[i + 1]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,Circumscribe(gpu_rect[0]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,Circumscribe(model_arr[i + 1]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,)
																											if i < 1:
																																		self.play(
																																		    MoveToTarget(model_cpu_arr[i]     )		,MoveToTarget(model_cpu_arr[i + 1]     )		,)
																											else:
																																		self.play(
																																		    MoveToTarget(model_cpu_arr[i]		,run_time=0.7     )		,MoveToTarget(model_cpu_arr[i + 1]		,run_time=0.7     )		,)
																				else:
																											model_cpu_arr[i].generate_target()
																											model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]     )
																											input.generate_target()
																											input.target.next_to(model_arr[-1].get_right()		,RIGHT + 0.0_2		,buff=0.2     )

																											self.play(
																											    Circumscribe(model_arr[-1]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,Circumscribe(cpu_left_col_base[-1]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,Circumscribe(gpu_rect[0]		,color=__UpperCAmelCase		,**__UpperCAmelCase     )		,)

																											self.play(MoveToTarget(model_cpu_arr[i]     )     )

																				A__	     =							a_c
																				A__	     =							a_c.copy()

													input.generate_target()
													input.target.next_to(model_base[-1]		,RIGHT + 0.0_2		,buff=0.5     )
													self.play(
													    FadeOut(__UpperCAmelCase     )		,FadeOut(__UpperCAmelCase		,run_time=0.5     )		,)

													A__	     =							MarkupText(f'''Inference on a model too large for GPU memory\nis successfully completed.'''		,font_size=24     )
													step_a.move_to([2, 2, 0]     )

													self.play(Write(__UpperCAmelCase		,run_time=3     )		,MoveToTarget(__UpperCAmelCase     )     )

													self.wait()





