


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				A      = '''pt'''
elif is_tf_available():
				A      = '''tf'''
else:
				A      = '''jax'''


class     lowercase__  (					snake_case__					,							unittest.TestCase		):
		A__=  ByTaTokenizer
		A__=  False






		def 			_UpperCAmelCase  (					self      :					Union[str, Any] ):




									"""simple docstring"""







									super().setUp()
									UpperCAmelCase__										=     ByTaTokenizer()
									tokenizer.save_pretrained(self.tmpdirname )






		@cached_property
		def 			_UpperCAmelCase  (					self      :					Dict ):




									"""simple docstring"""







									return ByTaTokenizer.from_pretrained("google/byt5-small" )






		def 			_UpperCAmelCase  (					self      :					Optional[int]		,						**_lowercase      :					Dict ):




									"""simple docstring"""







									return self.tokenizer_class.from_pretrained(self.tmpdirname		,						**_A )






		def 			_UpperCAmelCase  (					self      :					Any		,						_lowercase      :					int		,						_lowercase      :					Union[str, Any]=False		,						_lowercase      :					Union[str, Any]=20		,						_lowercase      :					Dict=5 ):




									"""simple docstring"""







									UpperCAmelCase__										=     []
									for i in range(len(_A ) ):
																try:
																							UpperCAmelCase__										=     tokenizer.decode([i]		,						clean_up_tokenization_spaces=_A )
																except UnicodeDecodeError:
																							pass
																toks.append((i, tok) )

									UpperCAmelCase__										=     list(filter(lambda _lowercase : re.match(r"^[ a-zA-Z]+$"		,						t[1] )		,						_A ) )
									UpperCAmelCase__										=     list(filter(lambda _lowercase : [t[0]] == tokenizer.encode(t[1]		,						add_special_tokens=_A )		,						_A ) )
									if max_length is not None and len(_A ) > max_length:
																UpperCAmelCase__										=     toks[:max_length]
									if min_length is not None and len(_A ) < min_length and len(_A ) > 0:
																while len(_A ) < min_length:
																							UpperCAmelCase__										=     toks + toks
        # toks_str = [t[1] for t in toks]
									UpperCAmelCase__										=     [t[0] for t in toks]

									# Ensure consistency
									UpperCAmelCase__										=     tokenizer.decode(_A		,						clean_up_tokenization_spaces=_A )
									if " " not in output_txt and len(_A ) > 1:
																UpperCAmelCase__										=     (
																    tokenizer.decode([toks_ids[0]]		,						clean_up_tokenization_spaces=_A )
																    + ' '
																    + tokenizer.decode(toks_ids[1:]		,						clean_up_tokenization_spaces=_A )
																)
									if with_prefix_space:
																UpperCAmelCase__										=     ' ' + output_txt
									UpperCAmelCase__										=     tokenizer.encode(_A		,						add_special_tokens=_A )
									return output_txt, output_ids






		def 			_UpperCAmelCase  (					self      :					List[str] ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.ta_base_tokenizer
									UpperCAmelCase__										=     tokenizer(["hi</s>", "I went to the gym</s>", "</s>"] )
									UpperCAmelCase__										=     tokenizer(["hi", "I went to the gym", ""] )
									self.assertListEqual(batch_with_eos_added["input_ids"]		,						batch_without_eos_added["input_ids"] )






		def 			_UpperCAmelCase  (					self      :					Union[str, Any] ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.ta_base_tokenizer
									UpperCAmelCase__										=     'Unicode €.'
									UpperCAmelCase__										=     tokenizer(_A )
									UpperCAmelCase__										=     [88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
									self.assertEqual(encoded["input_ids"]		,						_A )

									# decoding
									UpperCAmelCase__										=     tokenizer.decode(_A )
									self.assertEqual(_A		,						"Unicode €.</s>" )

									UpperCAmelCase__										=     tokenizer("e è é ê ë" )
									UpperCAmelCase__										=     [1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
									self.assertEqual(encoded["input_ids"]		,						_A )
									# decoding
									UpperCAmelCase__										=     tokenizer.decode(_A )
									self.assertEqual(_A		,						"e è é ê ë</s>" )

									# encode/decode, but with `encode` instead of `__call__`
									self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë" ) )		,						"e è é ê ë</s>" )






		def 			_UpperCAmelCase  (					self      :					List[str] ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.ta_base_tokenizer
									UpperCAmelCase__										=     ['A long paragraph for summarization.', 'Another paragraph for summarization.']
									# fmt: off
									UpperCAmelCase__										=     [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
									# fmt: on
									UpperCAmelCase__										=     tokenizer(_A		,						padding=_A		,						return_tensors=_A )
									self.assertIsInstance(_A		,						_A )

									if FRAMEWORK != "jax":
																UpperCAmelCase__										=     list(batch.input_ids.numpy()[0] )
									else:
																UpperCAmelCase__										=     list(batch.input_ids.tolist()[0] )

									self.assertListEqual(_A		,						_A )

									self.assertEqual((2, 37)		,						batch.input_ids.shape )
									self.assertEqual((2, 37)		,						batch.attention_mask.shape )






		def 			_UpperCAmelCase  (					self      :					Optional[Any] ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.ta_base_tokenizer
									UpperCAmelCase__										=     ['A long paragraph for summarization.', 'Another paragraph for summarization.']
									UpperCAmelCase__										=     tokenizer(_A		,						padding=_A		,						return_tensors=_A )
									# check if input_ids are returned and no decoder_input_ids
									self.assertIn("input_ids"		,						_A )
									self.assertIn("attention_mask"		,						_A )
									self.assertNotIn("decoder_input_ids"		,						_A )
									self.assertNotIn("decoder_attention_mask"		,						_A )






		def 			_UpperCAmelCase  (					self      :					str ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.ta_base_tokenizer
									UpperCAmelCase__										=     [
									    'Summary of the text.',
									    'Another summary.',
									]
									UpperCAmelCase__										=     tokenizer(
									    text_target=_A		,						max_length=32		,						padding="max_length"		,						truncation=_A		,						return_tensors=_A )
									self.assertEqual(32		,						targets["input_ids"].shape[1] )






		def 			_UpperCAmelCase  (					self      :					Optional[Any] ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.ta_base_tokenizer
									UpperCAmelCase__										=     ['A long paragraph for summarization. </s>']
									UpperCAmelCase__										=     ['Summary of the text. </s>']
									# fmt: off
									UpperCAmelCase__										=     [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
									UpperCAmelCase__										=     [86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
									# fmt: on

									UpperCAmelCase__										=     tokenizer(_A		,						text_target=_A )

									self.assertEqual(_A		,						batch["input_ids"][0] )
									self.assertEqual(_A		,						batch["labels"][0] )






		def 			_UpperCAmelCase  (					self      :					str ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.get_tokenizers()
									for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}""" ):
																							self.assertNotEqual(tokenizer.model_max_length		,						42 )

        # Now let's start the test
									UpperCAmelCase__										=     self.get_tokenizers()
									for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}""" ):
																							# Isolate this from the other tests because we save additional tokens/etc
																							UpperCAmelCase__										=     tempfile.mkdtemp()

																							UpperCAmelCase__										=     ' He is very happy, UNwant\u00E9d,running'
																							UpperCAmelCase__										=     tokenizer.encode(_A		,						add_special_tokens=_A )
																							tokenizer.save_pretrained(_A )

																							UpperCAmelCase__										=     tokenizer.__class__.from_pretrained(_A )
																							UpperCAmelCase__										=     after_tokenizer.encode(_A		,						add_special_tokens=_A )
																							self.assertListEqual(_A		,						_A )

																							shutil.rmtree(_A )

									UpperCAmelCase__										=     self.get_tokenizers(model_max_length=42 )
									for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}""" ):
																							# Isolate this from the other tests because we save additional tokens/etc
																							UpperCAmelCase__										=     tempfile.mkdtemp()

																							UpperCAmelCase__										=     ' He is very happy, UNwant\u00E9d,running'
																							tokenizer.add_tokens(["bim", "bambam"] )
																							UpperCAmelCase__										=     tokenizer.additional_special_tokens
																							additional_special_tokens.append("new_additional_special_token" )
																							tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens} )
																							UpperCAmelCase__										=     tokenizer.encode(_A		,						add_special_tokens=_A )
																							tokenizer.save_pretrained(_A )

																							UpperCAmelCase__										=     tokenizer.__class__.from_pretrained(_A )
																							UpperCAmelCase__										=     after_tokenizer.encode(_A		,						add_special_tokens=_A )
																							self.assertListEqual(_A		,						_A )
																							self.assertIn("new_additional_special_token"		,						after_tokenizer.additional_special_tokens )
																							self.assertEqual(after_tokenizer.model_max_length		,						42 )

																							UpperCAmelCase__										=     tokenizer.__class__.from_pretrained(_A		,						model_max_length=43 )
																							self.assertEqual(tokenizer.model_max_length		,						43 )

																							shutil.rmtree(_A )






		def 			_UpperCAmelCase  (					self      :					Optional[int] ):




									"""simple docstring"""







									UpperCAmelCase__										=     []
									if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

									if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																							tokenizer_utils.save_pretrained(_A )

																							with open(os.path.join(_A		,						"special_tokens_map.json" )		,						encoding="utf-8" ) as json_file:
																														UpperCAmelCase__										=     json.load(_A )

																							with open(os.path.join(_A		,						"tokenizer_config.json" )		,						encoding="utf-8" ) as json_file:
																														UpperCAmelCase__										=     json.load(_A )

																							UpperCAmelCase__										=     [F"""<extra_id_{i}>""" for i in range(1_25 )]

																							UpperCAmelCase__										=     added_tokens_extra_ids + [
																							    'an_additional_special_token'
																							]
																							UpperCAmelCase__										=     added_tokens_extra_ids + [
																							    'an_additional_special_token'
																							]

																							with open(os.path.join(_A		,						"special_tokens_map.json" )		,						"w"		,						encoding="utf-8" ) as outfile:
																														json.dump(_A		,						_A )
																							with open(os.path.join(_A		,						"tokenizer_config.json" )		,						"w"		,						encoding="utf-8" ) as outfile:
																														json.dump(_A		,						_A )

																							# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																							# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																							# "special_tokens_map.json" files
																							UpperCAmelCase__										=     tokenizer_class.from_pretrained(
																							    _A		,						)
																							self.assertIn(
																							    "an_additional_special_token"		,						tokenizer_without_change_in_init.additional_special_tokens )
																							# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																							self.assertEqual(
																							    ["an_additional_special_token"]		,						tokenizer_without_change_in_init.convert_ids_to_tokens(
																							        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"] ) )		,						)

																							# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																							UpperCAmelCase__										=     added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"		,						lstrip=_A )]
																							UpperCAmelCase__										=     tokenizer_class.from_pretrained(
																							    _A		,						additional_special_tokens=_A		,						)

																							self.assertIn("a_new_additional_special_token"		,						tokenizer.additional_special_tokens )
																							self.assertEqual(
																							    ["a_new_additional_special_token"]		,						tokenizer.convert_ids_to_tokens(
																							        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"] ) )		,						)






		def 			_UpperCAmelCase  (					self      :					List[str] ):




									"""simple docstring"""







									UpperCAmelCase__										=     []
									if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

									if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																							tokenizer_utils.save_pretrained(_A )

																							UpperCAmelCase__										=     tokenizer_class.from_pretrained(_A )

																							self.assertTrue(tokenizer.decode([2_55] ) == "" )






		def 			_UpperCAmelCase  (					self      :					Optional[int] ):




									"""simple docstring"""







									pass






		def 			_UpperCAmelCase  (					self      :					str ):




									"""simple docstring"""







									pass






		def 			_UpperCAmelCase  (					self      :					List[Any] ):




									"""simple docstring"""







									pass






		def 			_UpperCAmelCase  (					self      :					Union[str, Any] ):




									"""simple docstring"""







									pass






		def 			_UpperCAmelCase  (					self      :					Dict ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.get_tokenizers(fast=_A		,						do_lower_case=_A )
									for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}""" ):
																							UpperCAmelCase__										=     ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
																							UpperCAmelCase__										=     tokenizer.convert_tokens_to_string(_A )

																							self.assertIsInstance(_A		,						_A )






		def 			_UpperCAmelCase  (					self      :					Any ):




									"""simple docstring"""







									UpperCAmelCase__										=     self.get_tokenizers()
									for tokenizer in tokenizers:
																with self.subTest(F"""{tokenizer.__class__.__name__}""" ):
																							UpperCAmelCase__										=     [
																							    'bos_token',
																							    'eos_token',
																							    'unk_token',
																							    'sep_token',
																							    'pad_token',
																							    'cls_token',
																							    'mask_token',
																							]

																							UpperCAmelCase__										=     0
																							UpperCAmelCase__										=     tokenizer.convert_ids_to_tokens(
																							    _A		,						skip_special_tokens=_A )

																							for attr in attributes_list:
																														setattr(_A		,						attr + "_id"		,						_A )
																														self.assertEqual(getattr(_A		,						_A )		,						_A )
																														self.assertEqual(getattr(_A		,						attr + "_id" )		,						_A )

																														setattr(_A		,						attr + "_id"		,						_A )
																														self.assertEqual(getattr(_A		,						_A )		,						_A )
																														self.assertEqual(getattr(_A		,						attr + "_id" )		,						_A )

																							setattr(_A		,						"additional_special_tokens_ids"		,						[] )
																							self.assertListEqual(getattr(_A		,						"additional_special_tokens" )		,						[] )
																							self.assertListEqual(getattr(_A		,						"additional_special_tokens_ids" )		,						[] )

																							setattr(_A		,						"additional_special_tokens_ids"		,						[token_id_to_test_setters] )
																							self.assertListEqual(getattr(_A		,						"additional_special_tokens" )		,						[token_to_test_setters] )
																							self.assertListEqual(getattr(_A		,						"additional_special_tokens_ids" )		,						[token_id_to_test_setters] )









import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


A      = get_tests_dir("fixtures/test_sentencepiece_bpe.model")


class     lowercase__  (					__SCREAMING_SNAKE_CASE					,							unittest.TestCase		):
  A__=  BartphoTokenizer
  A__=  False
  A__=  True






  def 			_UpperCAmelCase  (					self      :					Dict ):




         """simple docstring"""







         super().setUp()

         UpperCAmelCase__										=     ["▁This", "▁is", "▁a", "▁t", "est"]
         UpperCAmelCase__										=     dict(zip(_lowercase		,						range(len(_lowercase ) ) ) )
         UpperCAmelCase__										=     {"unk_token": "<unk>"}

         UpperCAmelCase__										=     os.path.join(self.tmpdirname		,						VOCAB_FILES_NAMES["monolingual_vocab_file"] )
         with open(self.monolingual_vocab_file		,						"w"		,						encoding="utf-8" ) as fp:
                for token in vocab_tokens:
                       fp.write(F"""{token} {vocab_tokens[token]}\n""" )

         UpperCAmelCase__										=     BartphoTokenizer(_lowercase		,						self.monolingual_vocab_file		,						**self.special_tokens_map )
         tokenizer.save_pretrained(self.tmpdirname )






  def 			_UpperCAmelCase  (					self      :					Dict		,						**_lowercase      :					Optional[Any] ):




         """simple docstring"""







         kwargs.update(self.special_tokens_map )
         return BartphoTokenizer.from_pretrained(self.tmpdirname		,						**_lowercase )






  def 			_UpperCAmelCase  (					self      :					List[Any]		,						_lowercase      :					Union[str, Any] ):




         """simple docstring"""







         UpperCAmelCase__										=     "This is a là test"
         UpperCAmelCase__										=     "This is a<unk><unk> test"
         return input_text, output_text






  def 			_UpperCAmelCase  (					self      :					Optional[Any] ):




         """simple docstring"""







         UpperCAmelCase__										=     BartphoTokenizer(_lowercase		,						self.monolingual_vocab_file		,						**self.special_tokens_map )
         UpperCAmelCase__										=     "This is a là test"
         UpperCAmelCase__										=     "▁This ▁is ▁a ▁l à ▁t est".split()
         UpperCAmelCase__										=     tokenizer.tokenize(_lowercase )
         self.assertListEqual(_lowercase		,						_lowercase )

         UpperCAmelCase__										=     tokens + [tokenizer.unk_token]
         UpperCAmelCase__										=     [4, 5, 6, 3, 3, 7, 8, 3]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase )		,						_lowercase )





