





import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin







class      lowerCamelCase(						__snake_case			, unittest.TestCase	):
     '''simple docstring'''


     __magic_name__   =			ProphetNetTokenizer
     __magic_name__   =			False




     def        lowerCAmelCase__		(    self							):
         super().setUp()

         _A     =							[
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         _A     =							os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['vocab_file']							)
         with open(self.vocab_file				,     'w'				,     encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)




     def        lowerCAmelCase__		(    self				,     snake_case_							):
         _A     =							"UNwant\u00E9d,running"
         _A     =							"unwanted, running"
         return input_text, output_text




     def        lowerCAmelCase__		(    self							):
         _A     =							self.tokenizer_class(self.vocab_file							)

         _A     =							tokenizer.tokenize('UNwant\u00E9d,running'							)
         self.assertListEqual(a_				,     ['un', '##want', '##ed', ',', 'runn', '##ing']							)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a_							)				,     [9, 6, 7, 12, 10, 11]							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'							)				,     ['ah', '\u535A', '\u63A8', 'zz']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)				,     ['hello', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)				,     ['hello']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_				,     strip_accents=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)				,     ['hällo', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)				,     ['h\u00E9llo']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_				,     strip_accents=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)				,     ['hallo', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)				,     ['hello']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)				,     ['hallo', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)				,     ['hello']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)				,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_				,     strip_accents=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)				,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_				,     strip_accents=a_							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)				,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							BasicTokenizer(do_lower_case=a_				,     never_split=['[UNK]']							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'							)				,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']							)




     def        lowerCAmelCase__		(    self							):
         _A     =							["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         _A     =							{}
         for i, token in enumerate(a_							):
             _A     =							i
         _A     =							WordpieceTokenizer(vocab=a_				,     unk_token='[UNK]'							)

         self.assertListEqual(tokenizer.tokenize(''							)				,     []							)

         self.assertListEqual(tokenizer.tokenize('unwanted running'							)				,     ['un', '##want', '##ed', 'runn', '##ing']							)

         self.assertListEqual(tokenizer.tokenize('unwantedX running'							)				,     ['[UNK]', 'runn', '##ing']							)




     @require_torch
     def        lowerCAmelCase__		(    self							):
         _A     =							self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'							)

         _A     =							["A long paragraph for summarization.", "Another paragraph for summarization."]
         _A     =							[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
         _A     =							tokenizer(a_				,     padding=a_				,     return_tensors='pt'							)
         self.assertIsInstance(a_				,     a_							)
         _A     =							list(batch.input_ids.numpy()[0]							)
         self.assertListEqual(a_				,     a_							)

         self.assertEqual((2, 9)				,     batch.input_ids.shape							)
         self.assertEqual((2, 9)				,     batch.attention_mask.shape							)




     def        lowerCAmelCase__		(    self							):
         self.assertTrue(_is_whitespace(' '							)							)
         self.assertTrue(_is_whitespace('\t'							)							)
         self.assertTrue(_is_whitespace('\r'							)							)
         self.assertTrue(_is_whitespace('\n'							)							)
         self.assertTrue(_is_whitespace('\u00A0'							)							)

         self.assertFalse(_is_whitespace('A'							)							)
         self.assertFalse(_is_whitespace('-'							)							)




     def        lowerCAmelCase__		(    self							):
         self.assertTrue(_is_control('\u0005'							)							)

         self.assertFalse(_is_control('A'							)							)
         self.assertFalse(_is_control(' '							)							)
         self.assertFalse(_is_control('\t'							)							)
         self.assertFalse(_is_control('\r'							)							)




     def        lowerCAmelCase__		(    self							):
         self.assertTrue(_is_punctuation('-'							)							)
         self.assertTrue(_is_punctuation('$'							)							)
         self.assertTrue(_is_punctuation('`'							)							)
         self.assertTrue(_is_punctuation('.'							)							)

         self.assertFalse(_is_punctuation('A'							)							)
         self.assertFalse(_is_punctuation(' '							)							)




     @slow
     def        lowerCAmelCase__		(    self							):
         _A     =							self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'							)

         _A     =							tokenizer.encode('sequence builders'				,     add_special_tokens=a_							)
         _A     =							tokenizer.encode('multi-sequence build'				,     add_special_tokens=a_							)

         _A     =							tokenizer.build_inputs_with_special_tokens(a_							)
         _A     =							tokenizer.build_inputs_with_special_tokens(a_				,     a_							)

         assert encoded_sentence == text + [102]
         assert encoded_pair == text + [102] + text_a + [102]









'''simple docstring'''
from manim import *




class        _UpperCamelCase	(    SCREAMING_SNAKE_CASE):


				'''simple docstring'''


				def 	a__		(     self							)    ->  List[str]:
					lowercase						:		List[Any]   					=		Rectangle(height=0.5							,      width=0.5							)
					lowercase						:		str   					=		Rectangle(height=0.25							,      width=0.25							)
					lowercase						:		Union[str, Any]   					=		Rectangle(height=0.46							,      width=0.46							).set_stroke(width=0							)

					lowercase						:		List[str]   					=		[mem.copy() for i in range(6							)]
					lowercase						:		Any   					=		[mem.copy() for i in range(6							)]
					lowercase						:		List[str]   					=		VGroup(*a_							).arrange(a_							,      buff=0							)
					lowercase						:		List[Any]   					=		VGroup(*a_							).arrange(a_							,      buff=0							)
					lowercase						:		List[Any]   					=		VGroup(a_							,      a_							).arrange(a_							,      buff=0							)
					lowercase						:		Union[str, Any]   					=		Text("CPU"							,      font_size=2_4							)
					lowercase						:		List[Any]   					=		Group(a_							,      a_							).arrange(a_							,      buff=0.5							,      aligned_edge=a_							)
					cpu.move_to([-2.5, -0.5, 0]							)
					self.add(a_							)

					lowercase						:		List[Any]   					=		[mem.copy() for i in range(4							)]
					lowercase						:		Union[str, Any]   					=		VGroup(*a_							).arrange(a_							,      buff=0							)
					lowercase						:		Dict   					=		Text("GPU"							,      font_size=2_4							)
					lowercase						:		Tuple   					=		Group(a_							,      a_							).arrange(a_							,      buff=0.5							,      aligned_edge=a_							)
					gpu.move_to([-1, -1, 0]							)
					self.add(a_							)

					lowercase						:		Tuple   					=		[mem.copy() for i in range(6							)]
					lowercase						:		Optional[int]   					=		VGroup(*a_							).arrange(a_							,      buff=0							)

					lowercase						:		Any   					=		Text("Model"							,      font_size=2_4							)
					lowercase						:		str   					=		Group(a_							,      a_							).arrange(a_							,      buff=0.5							,      aligned_edge=a_							)
					model.move_to([3, -1.0, 0]							)
					self.add(a_							)

					lowercase						:		Dict   					=		[]
					lowercase						:		Tuple   					=		[]
					lowercase						:		List[Any]   					=		[]

					for i, rect in enumerate(a_							):
						rect.set_stroke(a_							)

						lowercase						:		Tuple   					=		Rectangle(height=0.46 / 4							,      width=0.46 / 3							).set_stroke(width=0.0							).set_fill(a_							,      opacity=0.7							)

						if i == 0:
							cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)							,      buff=0.02							,      direction=a_							)
							cpu_target.set_x(cpu_target.get_x() + 0.1							)
						elif i == 3:
							cpu_target.next_to(model_cpu_arr[0]							,      direction=a_							,      buff=0.0							)
						else:
							cpu_target.next_to(model_cpu_arr[i - 1]							,      direction=a_							,      buff=0.0							)
						self.add(a_							)
						model_cpu_arr.append(a_							)

					self.add(*a_							,      *a_							,      *a_							)

					lowercase						:		Any   					=		[mem.copy() for i in range(6							)]
					lowercase						:		Dict   					=		VGroup(*a_							).arrange(a_							,      buff=0							)

					lowercase						:		List[str]   					=		Text("Loaded Checkpoint"							,      font_size=2_4							)
					lowercase						:		Optional[Any]   					=		Group(a_							,      a_							).arrange(a_							,      buff=0.5							,      aligned_edge=a_							)
					checkpoint.move_to([3, 0.5, 0]							)

					self.add(a_							)

					lowercase						:		Any   					=		[]
					lowercase						:		int   					=		[]

					for i, rect in enumerate(a_							):
						lowercase						:		str   					=		fill.copy().set_fill(a_							,      opacity=0.7							)
						target.move_to(a_							)
						ckpt_arr.append(a_							)

						lowercase						:		Optional[Any]   					=		target.copy()
						if i < 5:
							cpu_target.move_to(cpu_left_col_base[i + 1]							)
						else:
							cpu_target.move_to(cpu_right_col_base[i - 5]							)
						ckpt_cpu_arr.append(a_							)
					self.add(*a_							,      *a_							)

					lowercase						:		Optional[Any]   					=		Square(side_length=2.2							)
					key.move_to([-5, 2, 0]							)

					lowercase						:		str   					=		MarkupText(
					    F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''							,      font_size=1_8							,      )

					key_text.move_to([-5, 2.4, 0]							)

					self.add(a_							,      a_							)

					lowercase						:		Any   					=		MarkupText(
					    F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''							,      font_size=1_8							,      )

					blue_text.next_to(a_							,      DOWN * 2.4							,      aligned_edge=key_text.get_left()							)
					self.add(a_							)

					lowercase						:		List[Any]   					=		MarkupText(
					    F'''Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'''							,      font_size=2_4							,      )
					step_a.move_to([2, 2, 0]							)

					lowercase						:		Any   					=		[meta_mem.copy() for i in range(6							)]
					lowercase						:		Dict   					=		[meta_mem.copy() for i in range(6							)]
					lowercase						:		Union[str, Any]   					=		VGroup(*a_							).arrange(a_							,      buff=0							)
					lowercase						:		Dict   					=		VGroup(*a_							).arrange(a_							,      buff=0							)
					lowercase						:		Any   					=		VGroup(a_							,      a_							).arrange(a_							,      buff=0							)
					lowercase						:		Optional[Any]   					=		Text("Disk"							,      font_size=2_4							)
					lowercase						:		List[str]   					=		Group(a_							,      a_							).arrange(a_							,      buff=0.5							,      aligned_edge=a_							)
					disk.move_to([-4.0, -1.25, 0]							)
					self.play(Write(a_							,      run_time=3							)							,      Write(a_							,      run_time=1							)							,      Create(a_							,      run_time=1							)							)

					lowercase						:		Optional[Any]   					=		[]
					for i, rect in enumerate(a_							):
						lowercase						:		int   					=		rect.copy()
						target.generate_target()
						target.target.move_to(disk_left_col_base[i]							).scale(0.5							)
						animations.append(MoveToTarget(a_							,      run_time=1.5							)							)
					self.play(*a_							)

					self.play(FadeOut(a_							)							)

					lowercase						:		List[Any]   					=		MarkupText(F'''Then, the checkpoint is removed from memory\nthrough garbage collection.'''							,      font_size=2_4							)
					step_a.move_to([2, 2, 0]							)

					self.play(Write(a_							,      run_time=3							)							)

					self.play(
					    FadeOut(a_							,      a_							,      *a_							,      *a_							)							,      )

					self.wait()


