




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class  __magic_name__						(  __snake_case    ,  unittest.TestCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							Dict	      =					GPTSanJapaneseTokenizer
				SCREAMING_SNAKE_CASE__					:							Any	      =					False
				SCREAMING_SNAKE_CASE__					:							Tuple	      =					{"do_clean_text": False, "add_prefix_space": False}


				def     _A							(							self:   str		):
											super().setUp()

											# fmt: off
											SCREAMING_SNAKE_CASE_   	=    ["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
											# fmt: on
											SCREAMING_SNAKE_CASE_   	=    {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
											SCREAMING_SNAKE_CASE_   	=    {"""unk_token""": """<unk>"""}

											SCREAMING_SNAKE_CASE_   	=    os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES['''vocab_file''']		)
											SCREAMING_SNAKE_CASE_   	=    os.path.join(self.tmpdirname		,					VOCAB_FILES_NAMES['''emoji_file''']		)
											with open(self.vocab_file		,					'''w'''		,					encoding='''utf-8'''		) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)
											with open(self.emoji_file		,					'''w'''		) as emoji_writer:
																		emoji_writer.write(json.dumps(_lowercase		)		)


				def     _A							(							self:   List[str]		,					**_lowerCamelCase:   Dict		):
											kwargs.update(self.special_tokens_map		)
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname		,					**_lowercase		)


				def     _A							(							self:   Dict		,					_lowerCamelCase:   List[Any]		):
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、世界。 \nこんばんは、㔺界。😀"""
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、世界。 \nこんばんは、世界。😀"""
											return input_text, output_text


				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   List[Any]		):
											SCREAMING_SNAKE_CASE_   	=    self.get_input_output_texts(_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode(_lowercase		,					add_special_tokens=_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.decode(_lowercase		,					clean_up_tokenization_spaces=_lowercase		)
											return text, ids


				def     _A							(							self:   List[str]		):
											pass  # TODO add if relevant


				def     _A							(							self:   Optional[int]		):
											pass  # TODO add if relevant


				def     _A							(							self:   List[str]		):
											pass  # TODO add if relevant


				def     _A							(							self:   Tuple		):
											SCREAMING_SNAKE_CASE_   	=    self.get_tokenizer()

											# Testing tokenization
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、世界。　こんばんは、㔺界。"""
											SCREAMING_SNAKE_CASE_   	=    ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
											SCREAMING_SNAKE_CASE_   	=    tokenizer.tokenize(_lowercase		)
											self.assertListEqual(_lowercase		,					_lowercase		)

											# Testing conversion to ids without special tokens
											SCREAMING_SNAKE_CASE_   	=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											SCREAMING_SNAKE_CASE_   	=    tokenizer.convert_tokens_to_ids(_lowercase		)
											self.assertListEqual(_lowercase		,					_lowercase		)

											# Testing conversion to ids with special tokens
											SCREAMING_SNAKE_CASE_   	=    tokens + [tokenizer.unk_token]
											SCREAMING_SNAKE_CASE_   	=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
											SCREAMING_SNAKE_CASE_   	=    tokenizer.convert_tokens_to_ids(_lowercase		)
											self.assertListEqual(_lowercase		,					_lowercase		)


				def     _A							(							self:   int		):
											SCREAMING_SNAKE_CASE_   	=    self.get_tokenizer()

											# Testing tokenization
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、、、、世界。こんばんは、、、、世界。"""
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode(_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.decode(_lowercase		)
											self.assertEqual(_lowercase		,					_lowercase		)


				@slow
				def     _A							(							self:   Any		):
											SCREAMING_SNAKE_CASE_   	=    self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''		)

											# Testing tokenization
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、世界。"""
											SCREAMING_SNAKE_CASE_   	=    """こんばんは、㔺界。😀"""
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、世界。こんばんは、世界。😀"""
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode(prefix_text + input_text		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode(''''''		,					prefix_text=prefix_text + input_text		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode(_lowercase		,					prefix_text=_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.decode(_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.decode(_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.decode(_lowercase		)
											self.assertEqual(_lowercase		,					_lowercase		)
											self.assertEqual(_lowercase		,					_lowercase		)
											self.assertEqual(_lowercase		,					_lowercase		)


				@slow
				def     _A							(							self:   Dict		):
											SCREAMING_SNAKE_CASE_   	=    self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''		)

											# Testing tokenization
											SCREAMING_SNAKE_CASE_   	=    """こんにちは、世界。"""
											SCREAMING_SNAKE_CASE_   	=    """こんばんは、㔺界。😀"""

											SCREAMING_SNAKE_CASE_   	=    len(tokenizer.encode(_lowercase		)		) - 2
											SCREAMING_SNAKE_CASE_   	=    len(tokenizer.encode(_lowercase		)		) - 2

											SCREAMING_SNAKE_CASE_   	=    [1] + [0] * (len_prefix + len_text + 1)
											SCREAMING_SNAKE_CASE_   	=    [1] * (len_prefix + len_text + 1) + [0]
											SCREAMING_SNAKE_CASE_   	=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											SCREAMING_SNAKE_CASE_   	=    tokenizer(prefix_text + input_text		).token_type_ids
											SCREAMING_SNAKE_CASE_   	=    tokenizer(''''''		,					prefix_text=prefix_text + input_text		).token_type_ids
											SCREAMING_SNAKE_CASE_   	=    tokenizer(_lowercase		,					prefix_text=_lowercase		).token_type_ids
											self.assertListEqual(_lowercase		,					_lowercase		)
											self.assertListEqual(_lowercase		,					_lowercase		)
											self.assertListEqual(_lowercase		,					_lowercase		)


				@slow
				def     _A							(							self:   Any		):
											SCREAMING_SNAKE_CASE_   	=    self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''		)

											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode('''あンいワ'''		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode(''''''		,					prefix_text='''あンいワ'''		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.encode('''いワ'''		,					prefix_text='''あン'''		)

											self.assertEqual(tokenizer.decode(_lowercase		)		,					tokenizer.decode(_lowercase		)		)
											self.assertEqual(tokenizer.decode(_lowercase		)		,					tokenizer.decode(_lowercase		)		)
											self.assertNotEqual(_lowercase		,					_lowercase		)
											self.assertNotEqual(_lowercase		,					_lowercase		)
											self.assertEqual(x_token_a[1]		,					x_token_a[-1]		)  # SEG token
											self.assertEqual(x_token_a[1]		,					x_token_a[3]		)  # SEG token


				@slow
				def     _A							(							self:   str		):
											SCREAMING_SNAKE_CASE_   	=    self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''		)

											SCREAMING_SNAKE_CASE_   	=    [["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
											SCREAMING_SNAKE_CASE_   	=    tokenizer(_lowercase		,					padding=_lowercase		)
											SCREAMING_SNAKE_CASE_   	=    tokenizer.batch_encode_plus(_lowercase		,					padding=_lowercase		)

											# fmt: off
											SCREAMING_SNAKE_CASE_   	=    [[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
											SCREAMING_SNAKE_CASE_   	=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											SCREAMING_SNAKE_CASE_   	=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids		,					_lowercase		)
											self.assertListEqual(x_token.token_type_ids		,					_lowercase		)
											self.assertListEqual(x_token.attention_mask		,					_lowercase		)
											self.assertListEqual(x_token_a.input_ids		,					_lowercase		)
											self.assertListEqual(x_token_a.token_type_ids		,					_lowercase		)
											self.assertListEqual(x_token_a.attention_mask		,					_lowercase		)


				def     _A							(							self:   List[str]		):
											pass





				def     _A							(							self:   List[str]		):
											pass








import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
			import torch

			from transformers.generation import DisjunctiveConstraint






@require_torch
class  __magic_name__						(  unittest.TestCase):
				'''simple docstring'''







				def     _A							(							self:   str		):
											# For consistency across different places the DisjunctiveConstraint is called,
											# dc.token_ids is a list of integers. It is also initialized only by integers.

											SCREAMING_SNAKE_CASE_   	=    [[1, 2, 4], [1, 2, 3, 4]]
											SCREAMING_SNAKE_CASE_   	=    DisjunctiveConstraint(_lowerCamelCase		)
											self.assertTrue(isinstance(dc.token_ids		,					_lowerCamelCase		)		)

											with self.assertRaises(_lowerCamelCase		):
																		DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]		)		)

											with self.assertRaises(_lowerCamelCase		):
																		DisjunctiveConstraint([torch.LongTensor([1, 2, 4]		), torch.LongTensor([1, 2, 3, 4, 5]		)]		)


				def     _A							(							self:   Union[str, Any]		):
											# We can't have constraints that are complete subsets of another. This leads to a preverse
											# interpretation of "constraint fulfillment": does generating [1,2,3] fulfill the constraint?
											# It would mean that it generated [1,2] which fulfills it, but it's in the middle of potentially
											# fulfilling [1,2,3,4]. If we believe that [1,2,3] does fulfill the constraint, then the algorithm
											# will necessarily never reach [1,2,3,4], giving users a false sense of control (better to just not allow it).
											SCREAMING_SNAKE_CASE_   	=    [[1, 2], [1, 2, 3, 4]]

											with self.assertRaises(_lowerCamelCase		):
																		DisjunctiveConstraint(_lowerCamelCase		)  # fails here


				def     _A							(							self:   Dict		):
											SCREAMING_SNAKE_CASE_   	=    [[1, 2, 3], [1, 2, 4]]

											SCREAMING_SNAKE_CASE_   	=    DisjunctiveConstraint(_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(1		)
											SCREAMING_SNAKE_CASE_   	=    stepped is True and completed is False and reset is False
											self.assertTrue(_lowerCamelCase		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.current_seq == [1]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(2		)
											SCREAMING_SNAKE_CASE_   	=    stepped is True and completed is False and reset is False
											self.assertTrue(_lowerCamelCase		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.current_seq == [1, 2]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(3		)
											SCREAMING_SNAKE_CASE_   	=    stepped is True and completed is True and reset is False
											self.assertTrue(_lowerCamelCase		)
											self.assertTrue(dc.completed		)  # Completed!
											self.assertTrue(dc.current_seq == [1, 2, 3]		)





				def     _A							(							self:   Optional[int]		):
											SCREAMING_SNAKE_CASE_   	=    [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

											SCREAMING_SNAKE_CASE_   	=    DisjunctiveConstraint(_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(1		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.current_seq == [1]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(2		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.current_seq == [1, 2]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(4		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.current_seq == [1, 2, 4]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(5		)
											self.assertTrue(dc.completed		)  # Completed!
											self.assertTrue(dc.current_seq == [1, 2, 4, 5]		)

											dc.reset()

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(1		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.remaining() == 3		)
											self.assertTrue(dc.current_seq == [1]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(2		)
											self.assertTrue(not dc.completed		)
											self.assertTrue(dc.remaining() == 2		)
											self.assertTrue(dc.current_seq == [1, 2]		)

											SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    dc.update(5		)
											self.assertTrue(dc.completed		)  # Completed!
											self.assertTrue(dc.remaining() == 0		)
											self.assertTrue(dc.current_seq == [1, 2, 5]		)


