




from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFFunnelBaseModel,
					    TFFunnelForMaskedLM,
					    TFFunnelForMultipleChoice,
					    TFFunnelForPreTraining,
					    TFFunnelForQuestionAnswering,
					    TFFunnelForSequenceClassification,
					    TFFunnelForTokenClassification,
					    TFFunnelModel,
					)







class        __lowerCAmelCase					:

			"""simple docstring"""







			def __init__( self     ,		lowerCamelCase__     ,		lowerCamelCase__=13     ,		lowerCamelCase__=7     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=99     ,		lowerCamelCase__=[1, 1, 2]     ,		lowerCamelCase__=1     ,		lowerCamelCase__=32     ,		lowerCamelCase__=4     ,		lowerCamelCase__=8     ,		lowerCamelCase__=37     ,		lowerCamelCase__="gelu_new"     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.0     ,		lowerCamelCase__=512     ,		lowerCamelCase__=3     ,		lowerCamelCase__=0.02     ,		lowerCamelCase__=3     ,		lowerCamelCase__=4     ,		lowerCamelCase__=None     ,		lowerCamelCase__=False     ,		)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       parent
							__lowerCamelCase									=       batch_size
							__lowerCamelCase									=       seq_length
							__lowerCamelCase									=       is_training
							__lowerCamelCase									=       use_input_mask
							__lowerCamelCase									=       use_token_type_ids
							__lowerCamelCase									=       use_labels
							__lowerCamelCase									=       vocab_size
							__lowerCamelCase									=       block_sizes
							__lowerCamelCase									=       num_decoder_layers
							__lowerCamelCase									=       d_model
							__lowerCamelCase									=       n_head
							__lowerCamelCase									=       d_head
							__lowerCamelCase									=       d_inner
							__lowerCamelCase									=       hidden_act
							__lowerCamelCase									=       hidden_dropout
							__lowerCamelCase									=       attention_dropout
							__lowerCamelCase									=       activation_dropout
							__lowerCamelCase									=       max_position_embeddings
							__lowerCamelCase									=       type_vocab_size
							__lowerCamelCase									=       2
							__lowerCamelCase									=       num_labels
							__lowerCamelCase									=       num_choices
							__lowerCamelCase									=       scope
							__lowerCamelCase									=       initializer_std

							# Used in the tests to check the size of the first attention layer
							__lowerCamelCase									=       n_head
							# Used in the tests to check the size of the first hidden state
							__lowerCamelCase									=       self.d_model
							# Used in the tests to check the number of output hidden states/attentions
							__lowerCamelCase									=       sum(self.block_sizes				) + (0 if base else self.num_decoder_layers)
							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
							# the last hidden state of the first block (which is the first hidden state of the decoder).
							if not base:
											__lowerCamelCase									=       self.num_hidden_layers + 2


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       ids_tensor([self.batch_size, self.seq_length]     ,		self.vocab_size				)

							__lowerCamelCase									=       None
							if self.use_input_mask:
											__lowerCamelCase									=       random_attention_mask([self.batch_size, self.seq_length]				)

							__lowerCamelCase									=       None
							if self.use_token_type_ids:
											__lowerCamelCase									=       ids_tensor([self.batch_size, self.seq_length]     ,		self.type_vocab_size				)

							__lowerCamelCase									=       None
							__lowerCamelCase									=       None
							__lowerCamelCase									=       None
							if self.use_labels:
											__lowerCamelCase									=       ids_tensor([self.batch_size]     ,		self.type_sequence_label_size				)
											__lowerCamelCase									=       ids_tensor([self.batch_size, self.seq_length]     ,		self.num_labels				)
											__lowerCamelCase									=       ids_tensor([self.batch_size]     ,		self.num_choices				)

							__lowerCamelCase									=       FunnelConfig(
							    vocab_size=self.vocab_size     ,		block_sizes=self.block_sizes     ,		num_decoder_layers=self.num_decoder_layers     ,		d_model=self.d_model     ,		n_head=self.n_head     ,		d_head=self.d_head     ,		d_inner=self.d_inner     ,		hidden_act=self.hidden_act     ,		hidden_dropout=self.hidden_dropout     ,		attention_dropout=self.attention_dropout     ,		activation_dropout=self.activation_dropout     ,		max_position_embeddings=self.max_position_embeddings     ,		type_vocab_size=self.type_vocab_size     ,		initializer_std=self.initializer_std     ,		)

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelModel(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)

							__lowerCamelCase									=       [input_ids, input_mask]
							__lowerCamelCase									=       model(__UpperCAmelCase				)

							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.d_model)				)

							__lowerCamelCase									=       False
							__lowerCamelCase									=       TFFunnelModel(config=__UpperCAmelCase				)
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.d_model)				)

							__lowerCamelCase									=       False
							__lowerCamelCase									=       TFFunnelModel(config=__UpperCAmelCase				)
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.d_model)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelBaseModel(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)

							__lowerCamelCase									=       [input_ids, input_mask]
							__lowerCamelCase									=       model(__UpperCAmelCase				)

							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, 2, self.d_model)				)

							__lowerCamelCase									=       False
							__lowerCamelCase									=       TFFunnelBaseModel(config=__UpperCAmelCase				)
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, 3, self.d_model)				)

							__lowerCamelCase									=       False
							__lowerCamelCase									=       TFFunnelBaseModel(config=__UpperCAmelCase				)
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, 2, self.d_model)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelForPreTraining(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelForMaskedLM(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.vocab_size)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       self.num_labels
							__lowerCamelCase									=       TFFunnelForSequenceClassification(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_labels)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       self.num_choices
							__lowerCamelCase									=       TFFunnelForMultipleChoice(config=__UpperCAmelCase				)
							__lowerCamelCase									=       tf.tile(tf.expand_dims(__UpperCAmelCase     ,		1				)     ,		(1, self.num_choices, 1)				)
							__lowerCamelCase									=       tf.tile(tf.expand_dims(__UpperCAmelCase     ,		1				)     ,		(1, self.num_choices, 1)				)
							__lowerCamelCase									=       tf.tile(tf.expand_dims(__UpperCAmelCase     ,		1				)     ,		(1, self.num_choices, 1)				)
							__lowerCamelCase									=       {
							    'input_ids': multiple_choice_inputs_ids,
							    'attention_mask': multiple_choice_input_mask,
							    'token_type_ids': multiple_choice_token_type_ids,
							}
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_choices)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       self.num_labels
							__lowerCamelCase									=       TFFunnelForTokenClassification(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.num_labels)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelForQuestionAnswering(config=__UpperCAmelCase				)
							__lowerCamelCase									=       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							__lowerCamelCase									=       model(__UpperCAmelCase				)
							self.parent.assertEqual(result.start_logits.shape     ,		(self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape     ,		(self.batch_size, self.seq_length)				)






			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       self.prepare_config_and_inputs()
							(
							    (
							    __lowerCamelCase		
							)			,		(
							    __lowerCamelCase		
							)			,		(
							    __lowerCamelCase		
							)			,		(
							    __lowerCamelCase		
							)			,		(
							    __lowerCamelCase		
							)			,		(
							    __lowerCamelCase		
							)			,		(
							    __lowerCamelCase		
							)			,				
							)									=       config_and_inputs
							__lowerCamelCase									=       {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict







@require_tf
class        __lowerCAmelCase					(			A__     ,							A__     ,							unittest.TestCase					):

			"""simple docstring"""







			snake_case_  						=			(
			    (
			        TFFunnelModel,
			        TFFunnelForMaskedLM,
			        TFFunnelForPreTraining,
			        TFFunnelForQuestionAnswering,
			        TFFunnelForTokenClassification,
			    )
			    if is_tf_available()
			    else ()
			)
			snake_case_  						=			(
			    {
			        'feature-extraction': (TFFunnelBaseModel, TFFunnelModel),
			        'fill-mask': TFFunnelForMaskedLM,
			        'question-answering': TFFunnelForQuestionAnswering,
			        'text-classification': TFFunnelForSequenceClassification,
			        'token-classification': TFFunnelForTokenClassification,
			        'zero-shot': TFFunnelForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			snake_case_  						=			False
			snake_case_  						=			False


			def  lowercase_       ( self				)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelModelTester(self				)
							__lowerCamelCase									=       ConfigTester(self     ,		config_class=__UpperCAmelCase				)


			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							self.config_tester.run_common_tests()


			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__UpperCAmelCase				)


			def  lowercase_       ( self				)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*__UpperCAmelCase				)


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase				)


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase				)






			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase				)







@require_tf
class        __lowerCAmelCase					(			A__     ,							unittest.TestCase					):

			"""simple docstring"""







			snake_case_  						=			(
			    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
			)
			snake_case_  						=			False
			snake_case_  						=			False


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       TFFunnelModelTester(self     ,		base=__UpperCAmelCase				)
							__lowerCamelCase									=       ConfigTester(self     ,		config_class=__UpperCAmelCase				)


			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''
							self.config_tester.run_common_tests()


			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_base_model(*__UpperCAmelCase				)


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase				)






			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase				)





"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Union[str, Any]       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		str       		=				False
			lowerCamelCase__ :		Optional[int]       		=				True

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     str:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """こんにちは""",
						    """こん""",
						    """にちは""",
						    """ばんは""",
						    """##こん""",
						    """##にちは""",
						    """##ばんは""",
						    """世界""",
						    """##世界""",
						    """、""",
						    """##、""",
						    """。""",
						    """##。""",
						]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]     ,					__UpperCAmelCase					:    Optional[int]							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     int:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.get_input_output_texts(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.decode(__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase							)
						return text, ids

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Tuple:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""mecab"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Union[str, Any]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic_lite"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(do_lower_case=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     List[str]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(
									    do_lower_case=__UpperCAmelCase     ,					normalize_text=__UpperCAmelCase     ,					mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""							)
						except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(normalize_text=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""sudachi"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""A"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国""", """人""", """参政""", """権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""B"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人""", """参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""C"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(do_lower_case=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     str:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(normalize_text=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(trim_whitespace=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""jumanpp"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(do_lower_case=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(normalize_text=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(trim_whitespace=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""							)     ,					["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   WordpieceTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こんにちは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは"""							)     ,					["""こん""", """##ばんは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""							)     ,					["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.subword_tokenizer

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]							)

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Dict       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		Tuple       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]     ,					**__UpperCAmelCase					:    List[Any]							) ->     List[Any]:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     ,					subword_tokenizer_type="""character"""     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    Any							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Union[str, Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					subword_tokenizer_type="""character"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""							)
						self.assertListEqual(
						    __UpperCAmelCase     ,					["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]							)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   CharacterTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こ""", """ん""", """に""", """ち""", """は"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちほ"""							)     ,					["""こ""", """ん""", """に""", """ち""", """[UNK]"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     str:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						SCREAMING_SNAKE_CASE__										=   AutoTokenizer.from_pretrained(__UpperCAmelCase							)
						self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase							)





class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)
						SCREAMING_SNAKE_CASE__										=   """bert-base-cased"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertJapaneseTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)


