





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import numpy
						import tensorflow as tf

						from transformers import (
						    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
						    BertConfig,
						    DPRConfig,
						    TFDPRContextEncoder,
						    TFDPRQuestionEncoder,
						    TFDPRReader,
						)
class SCREAMING_SNAKE_CASE_						:






						def __init__(      self      :      Optional[int]		,   _A      :      List[Any]		,   _A      :      str=13		,   _A      :      List[str]=7		,   _A      :      Dict=True		,   _A      :      Optional[Any]=True		,   _A      :      Dict=True		,   _A      :      int=True		,   _A      :      str=99		,   _A      :      Optional[Any]=32		,   _A      :      Optional[Any]=2		,   _A      :      Optional[int]=4		,   _A      :      Tuple=37		,   _A      :      Tuple="gelu"		,   _A      :      List[str]=0.1		,   _A      :      Union[str, Any]=0.1		,   _A      :      Any=512		,   _A      :      List[Any]=16		,   _A      :      List[str]=2		,   _A      :      int=0.0_2		,   _A      :      Any=3		,   _A      :      Optional[Any]=4		,   _A      :      int=None		,   _A      :      Any=0		,   )							->    Dict:







							"""simple docstring"""


							snake_case_		:			int											=	parent
							snake_case_		:			Tuple											=	batch_size
							snake_case_		:			Optional[Any]											=	seq_length
							snake_case_		:			Tuple											=	is_training
							snake_case_		:			List[str]											=	use_input_mask
							snake_case_		:			List[Any]											=	use_token_type_ids
							snake_case_		:			int											=	use_labels
							snake_case_		:			Dict											=	vocab_size
							snake_case_		:			str											=	hidden_size
							snake_case_		:			Tuple											=	num_hidden_layers
							snake_case_		:			Tuple											=	num_attention_heads
							snake_case_		:			Tuple											=	intermediate_size
							snake_case_		:			List[Any]											=	hidden_act
							snake_case_		:			Tuple											=	hidden_dropout_prob
							snake_case_		:			int											=	attention_probs_dropout_prob
							snake_case_		:			Dict											=	max_position_embeddings
							snake_case_		:			List[Any]											=	type_vocab_size
							snake_case_		:			Any											=	type_sequence_label_size
							snake_case_		:			int											=	initializer_range
							snake_case_		:			Union[str, Any]											=	num_labels
							snake_case_		:			Optional[int]											=	num_choices
							snake_case_		:			Optional[Any]											=	scope
							snake_case_		:			Union[str, Any]											=	projection_dim






						def UpperCAmelCase_				(      self      :      List[str]	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			Tuple											=	ids_tensor([self.batch_size, self.seq_length]		,   self.vocab_size	)

							snake_case_		:			Tuple											=	None
							if self.use_input_mask:
								# follow test_modeling_tf_ctrl.py
								snake_case_		:			Dict											=	random_attention_mask([self.batch_size, self.seq_length]	)

							snake_case_		:			List[str]											=	None
							if self.use_token_type_ids:
								snake_case_		:			Optional[Any]											=	ids_tensor([self.batch_size, self.seq_length]		,   self.type_vocab_size	)

							snake_case_		:			Any											=	None
							snake_case_		:			Optional[int]											=	None
							snake_case_		:			Dict											=	None
							if self.use_labels:
								snake_case_		:			Tuple											=	ids_tensor([self.batch_size]		,   self.type_sequence_label_size	)
								snake_case_		:			Optional[Any]											=	ids_tensor([self.batch_size, self.seq_length]		,   self.num_labels	)
								snake_case_		:			int											=	ids_tensor([self.batch_size]		,   self.num_choices	)

							snake_case_		:			Any											=	BertConfig(
							    vocab_size=self.vocab_size		,   hidden_size=self.hidden_size		,   num_hidden_layers=self.num_hidden_layers		,   num_attention_heads=self.num_attention_heads		,   intermediate_size=self.intermediate_size		,   hidden_act=self.hidden_act		,   hidden_dropout_prob=self.hidden_dropout_prob		,   attention_probs_dropout_prob=self.attention_probs_dropout_prob		,   max_position_embeddings=self.max_position_embeddings		,   type_vocab_size=self.type_vocab_size		,   is_decoder=_A		,   initializer_range=self.initializer_range		,   )
							snake_case_		:			Dict											=	DPRConfig(projection_dim=self.projection_dim		,   **config.to_dict()	)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def UpperCAmelCase_				(      self      :      Tuple		,   _A      :      Any		,   _A      :      str		,   _A      :      List[Any]		,   _A      :      str		,   _A      :      Dict		,   _A      :      Union[str, Any]		,   _A      :      int	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			str											=	TFDPRContextEncoder(config=_A	)
							snake_case_		:			Dict											=	model(_A		,   attention_mask=_A		,   token_type_ids=_A	)
							snake_case_		:			Optional[int]											=	model(_A		,   token_type_ids=_A	)
							snake_case_		:			str											=	model(_A	)
							self.parent.assertEqual(result.pooler_output.shape		,   (self.batch_size, self.projection_dim or self.hidden_size)	)






						def UpperCAmelCase_				(      self      :      Tuple		,   _A      :      List[str]		,   _A      :      Optional[Any]		,   _A      :      Union[str, Any]		,   _A      :      Tuple		,   _A      :      Dict		,   _A      :      str		,   _A      :      List[str]	)							->    Tuple:







							"""simple docstring"""


							snake_case_		:			Tuple											=	TFDPRQuestionEncoder(config=_A	)
							snake_case_		:			Optional[int]											=	model(_A		,   attention_mask=_A		,   token_type_ids=_A	)
							snake_case_		:			List[Any]											=	model(_A		,   token_type_ids=_A	)
							snake_case_		:			Optional[int]											=	model(_A	)
							self.parent.assertEqual(result.pooler_output.shape		,   (self.batch_size, self.projection_dim or self.hidden_size)	)






						def UpperCAmelCase_				(      self      :      int		,   _A      :      List[Any]		,   _A      :      List[str]		,   _A      :      Dict		,   _A      :      Optional[Any]		,   _A      :      Any		,   _A      :      Union[str, Any]		,   _A      :      Optional[Any]	)							->    str:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	TFDPRReader(config=_A	)
							snake_case_		:			Dict											=	model(_A		,   attention_mask=_A	)

							self.parent.assertEqual(result.start_logits.shape		,   (self.batch_size, self.seq_length)	)
							self.parent.assertEqual(result.end_logits.shape		,   (self.batch_size, self.seq_length)	)
							self.parent.assertEqual(result.relevance_logits.shape		,   (self.batch_size,)	)






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			int											=	self.prepare_config_and_inputs()
							(
							    (
							    snake_case_
							)						,(
							    snake_case_
							)						,(
							    snake_case_
							)						,(
							    snake_case_
							)						,(
							    snake_case_
							)						,(
							    snake_case_
							)						,(
							    snake_case_
							)						,
							)		:			Dict											=	config_and_inputs
							snake_case_		:			int											=	{'input_ids': input_ids}
							return config, inputs_dict
@require_tf
class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				snake_case_       ,				unittest.TestCase					):
						__magic_name__:			Optional[Any]       =       (
						    (
						        TFDPRContextEncoder,
						        TFDPRQuestionEncoder,
						        TFDPRReader,
						    )
						    if is_tf_available()
						    else ()
						)
						__magic_name__:			Optional[Any]       =       {"feature-extraction": TFDPRQuestionEncoder} if is_tf_available() else {}

						__magic_name__:			List[Any]       =       False
						__magic_name__:			Any       =       False
						__magic_name__:			Any       =       False
						__magic_name__:			List[str]       =       False
						__magic_name__:			int       =       False






						def UpperCAmelCase_				(      self      :      int	)							->    Any:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	TFDPRModelTester(self	)
							snake_case_		:			str											=	ConfigTester(self		,   config_class=_A		,   hidden_size=37	)






						def UpperCAmelCase_				(      self      :      str	)							->    Any:







							"""simple docstring"""


							self.config_tester.run_common_tests()






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			int											=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_context_encoder(*_A	)






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			List[str]											=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_question_encoder(*_A	)






						def UpperCAmelCase_				(      self      :      Any	)							->    str:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_reader(*_A	)






						@slow
						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    Tuple:







							"""simple docstring"""


							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		:			List[Any]											=	TFDPRContextEncoder.from_pretrained(_A	)
								self.assertIsNotNone(_A	)

							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		:			Optional[int]											=	TFDPRContextEncoder.from_pretrained(_A	)
								self.assertIsNotNone(_A	)

							for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		:			int											=	TFDPRQuestionEncoder.from_pretrained(_A	)
								self.assertIsNotNone(_A	)

							for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		:			List[Any]											=	TFDPRReader.from_pretrained(_A	)
								self.assertIsNotNone(_A	)
@require_tf
class SCREAMING_SNAKE_CASE_						(					unittest.TestCase					):






						@slow
						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    Tuple:







							"""simple docstring"""


							snake_case_		:			Any											=	TFDPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base'	)

							snake_case_		:			Union[str, Any]											=	tf.constant(
							    [[101, 7592, 1010, 2003, 2026, 3899, 10140, 1029, 102]]	)  # [CLS] hello, is my dog cute? [SEP]
							snake_case_		:			Optional[Any]											=	model(_A	)[0]  # embedding shape = (1, 768)
							# compare the actual values for a slice.
							snake_case_		:			List[str]											=	tf.constant(
							    [
							        [
							            0.0_3_2_3_6_2_5_3,
							            0.1_2_7_5_3_3_3_5,
							            0.1_6_8_1_8_5_0_9,
							            0.0_0_2_7_9_7_8_6,
							            0.3_8_9_6_9_3_3,
							            0.2_4_2_6_4_9_4_5,
							            0.2_1_7_8_9_7_1,
							            -0.0_2_3_3_5_2_2_7,
							            -0.0_8_4_8_1_9_5_9,
							            -0.1_4_3_2_4_1_1_7,
							        ]
							    ]	)
							self.assertTrue(numpy.allclose(output[:, :10].numpy()		,   expected_slice.numpy()		,   atol=1E-4	)	)








import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				unittest.TestCase					):
						__magic_name__:			Optional[Any]       =       BertJapaneseTokenizer
						__magic_name__:			Optional[int]       =       False
						__magic_name__:			Optional[int]       =       True






						def UpperCAmelCase_				(      self      :      Any	)							->    List[Any]:







							"""simple docstring"""


							super().setUp()

							snake_case_		:			int											=	[
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    'こんにちは',
							    'こん',
							    'にちは',
							    'ばんは',
							    '##こん',
							    '##にちは',
							    '##ばんは',
							    '世界',
							    '##世界',
							    '、',
							    '##、',
							    '。',
							    '##。',
							]

							snake_case_		:			List[Any]											=	os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['vocab_file']	)
							with open(self.vocab_file		,   'w'		,   encoding='utf-8'	) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)






						def UpperCAmelCase_				(      self      :      int		,   _A      :      Union[str, Any]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	'こんにちは、世界。 \nこんばんは、世界。'
							snake_case_		:			Optional[int]											=	'こんにちは 、 世界 。 こんばんは 、 世界 。'
							return input_text, output_text






						def UpperCAmelCase_				(      self      :      Any		,   _A      :      List[Any]	)							->    int:







							"""simple docstring"""


							snake_case_						,snake_case_		:			Optional[Any]											=	self.get_input_output_texts(_A	)
							snake_case_		:			Any											=	tokenizer.encode(_A		,   add_special_tokens=_A	)
							snake_case_		:			Union[str, Any]											=	tokenizer.decode(_A		,   clean_up_tokenization_spaces=_A	)
							return text, ids






						def UpperCAmelCase_				(      self      :      List[str]	)							->    str:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Dict	)							->    List[Any]:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      str	)							->    Optional[int]:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Tuple	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			List[str]											=	self.tokenizer_class(self.vocab_file	)

							snake_case_		:			int											=	tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			int											=	self.tokenizer_class(self.vocab_file		,   word_tokenizer_type='mecab'	)
							self.assertIsNotNone(_A	)

							snake_case_		:			str											=	'こんにちは、世界。\nこんばんは、世界。'
							snake_case_		:			Optional[Any]											=	tokenizer.tokenize(_A	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

							snake_case_		:			Optional[Any]											=	os.path.join(self.tmpdirname		,   'tokenizer.bin'	)
							with open(_A		,   'wb'	) as handle:
								pickle.dump(_A		,   _A	)

							with open(_A		,   'rb'	) as handle:
								snake_case_		:			Any											=	pickle.load(_A	)

							snake_case_		:			Optional[Any]											=	tokenizer_new.tokenize(_A	)

							self.assertListEqual(_A		,   _A	)






						def UpperCAmelCase_				(      self      :      Tuple	)							->    int:







							"""simple docstring"""


							snake_case_		:			str											=	MecabTokenizer(mecab_dic='ipadic'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Tuple	)							->    Optional[Any]:







							"""simple docstring"""


							try:
								snake_case_		:			Dict											=	MecabTokenizer(mecab_dic='unidic_lite'	)
							except ModuleNotFoundError:
								return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Tuple:







							"""simple docstring"""


							try:
								snake_case_		:			Any											=	MecabTokenizer(mecab_dic='unidic'	)
							except ModuleNotFoundError:
								return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Any	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Tuple											=	MecabTokenizer(do_lower_case=_A		,   mecab_dic='ipadic'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    int:







							"""simple docstring"""


							try:
								snake_case_		:			Union[str, Any]											=	MecabTokenizer(
								    do_lower_case=_A		,   normalize_text=_A		,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'	)
							except RuntimeError:
								# if dict doesn't exist in the system, previous code raises this error.
								return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	MecabTokenizer(normalize_text=_A		,   mecab_dic='ipadic'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    int:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	self.tokenizer_class(self.vocab_file		,   word_tokenizer_type='sudachi'	)
							self.assertIsNotNone(_A	)

							snake_case_		:			Optional[Any]											=	'こんにちは、世界。\nこんばんは、世界。'
							snake_case_		:			Optional[int]											=	tokenizer.tokenize(_A	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

							snake_case_		:			Optional[int]											=	os.path.join(self.tmpdirname		,   'tokenizer.bin'	)
							with open(_A		,   'wb'	) as handle:
								pickle.dump(_A		,   _A	)

							with open(_A		,   'rb'	) as handle:
								snake_case_		:			Dict											=	pickle.load(_A	)

							snake_case_		:			Any											=	tokenizer_new.tokenize(_A	)

							self.assertListEqual(_A		,   _A	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      str	)							->    Optional[Any]:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	SudachiTokenizer(sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    Optional[Any]:







							"""simple docstring"""


							snake_case_		:			str											=	SudachiTokenizer(sudachi_dict_type='core'		,   sudachi_split_mode='A'	)

							self.assertListEqual(tokenizer.tokenize('外国人参政権'	)		,   ['外国', '人', '参政', '権']	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Tuple	)							->    int:







							"""simple docstring"""


							snake_case_		:			List[str]											=	SudachiTokenizer(sudachi_dict_type='core'		,   sudachi_split_mode='B'	)

							self.assertListEqual(tokenizer.tokenize('外国人参政権'	)		,   ['外国人', '参政権']	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Any	)							->    int:







							"""simple docstring"""


							snake_case_		:			List[str]											=	SudachiTokenizer(sudachi_dict_type='core'		,   sudachi_split_mode='C'	)

							self.assertListEqual(tokenizer.tokenize('外国人参政権'	)		,   ['外国人参政権']	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    int:







							"""simple docstring"""


							snake_case_		:			Any											=	SudachiTokenizer(do_lower_case=_A		,   sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Any	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			Optional[int]											=	SudachiTokenizer(normalize_text=_A		,   sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      List[Any]	)							->    Dict:







							"""simple docstring"""


							snake_case_		:			List[str]											=	SudachiTokenizer(trim_whitespace=_A		,   sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      List[str]	)							->    Any:







							"""simple docstring"""


							snake_case_		:			Tuple											=	self.tokenizer_class(self.vocab_file		,   word_tokenizer_type='jumanpp'	)
							self.assertIsNotNone(_A	)

							snake_case_		:			Any											=	'こんにちは、世界。\nこんばんは、世界。'
							snake_case_		:			Any											=	tokenizer.tokenize(_A	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

							snake_case_		:			Any											=	os.path.join(self.tmpdirname		,   'tokenizer.bin'	)
							with open(_A		,   'wb'	) as handle:
								pickle.dump(_A		,   _A	)

							with open(_A		,   'rb'	) as handle:
								snake_case_		:			Optional[Any]											=	pickle.load(_A	)

							snake_case_		:			Union[str, Any]											=	tokenizer_new.tokenize(_A	)

							self.assertListEqual(_A		,   _A	)






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    int:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      str	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			Dict											=	JumanppTokenizer(do_lower_case=_A	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Tuple	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			str											=	JumanppTokenizer(normalize_text=_A	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			Optional[int]											=	JumanppTokenizer(trim_whitespace=_A	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Any	)							->    Any:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'	)		,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']		,   )






						def UpperCAmelCase_				(      self      :      int	)							->    Any:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

							snake_case_		:			List[Any]											=	{}
							for i, token in enumerate(_A	):
								snake_case_		:			List[str]											=	i
							snake_case_		:			List[str]											=	WordpieceTokenizer(vocab=_A		,   unk_token='[UNK]'	)

							self.assertListEqual(tokenizer.tokenize(''	)		,   []	)

							self.assertListEqual(tokenizer.tokenize('こんにちは'	)		,   ['こんにちは']	)

							self.assertListEqual(tokenizer.tokenize('こんばんは'	)		,   ['こん', '##ばんは']	)

							self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'	)		,   ['こん', '##ばんは', '[UNK]', 'こんにちは']	)






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Any											=	BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'	)
							snake_case_		:			Optional[int]											=	tokenizer.subword_tokenizer

							snake_case_		:			List[str]											=	subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'	)
							self.assertListEqual(_A		,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']	)

							snake_case_		:			Dict											=	subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'	)
							self.assertListEqual(_A		,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']	)






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Any:







							"""simple docstring"""


							snake_case_		:			Dict											=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'	)

							snake_case_		:			Tuple											=	tokenizer.encode('ありがとう。'		,   add_special_tokens=_A	)
							snake_case_		:			Optional[Any]											=	tokenizer.encode('どういたしまして。'		,   add_special_tokens=_A	)

							snake_case_		:			Dict											=	tokenizer.build_inputs_with_special_tokens(_A	)
							snake_case_		:			Optional[Any]											=	tokenizer.build_inputs_with_special_tokens(_A		,   _A	)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]
@custom_tokenizers
class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				unittest.TestCase					):
						__magic_name__:			List[Any]       =       BertJapaneseTokenizer
						__magic_name__:			int       =       False






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    List[Any]:







							"""simple docstring"""


							super().setUp()

							snake_case_		:			Optional[int]											=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							snake_case_		:			List[str]											=	os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['vocab_file']	)
							with open(self.vocab_file		,   'w'		,   encoding='utf-8'	) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)






						def UpperCAmelCase_				(      self      :      List[Any]		,   **_A      :      Dict	)							->    List[str]:







							"""simple docstring"""


							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname		,   subword_tokenizer_type='character'		,   **_A	)






						def UpperCAmelCase_				(      self      :      Dict		,   _A      :      str	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Tuple											=	'こんにちは、世界。 \nこんばんは、世界。'
							snake_case_		:			str											=	'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
							return input_text, output_text






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    int:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    str:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Tuple	)							->    str:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Dict:







							"""simple docstring"""


							snake_case_		:			Any											=	self.tokenizer_class(self.vocab_file		,   subword_tokenizer_type='character'	)

							snake_case_		:			int											=	tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'	)
							self.assertListEqual(
							    _A		,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']	)
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(_A	)		,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]	)






						def UpperCAmelCase_				(      self      :      Any	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			Dict											=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							snake_case_		:			Optional[Any]											=	{}
							for i, token in enumerate(_A	):
								snake_case_		:			str											=	i
							snake_case_		:			Tuple											=	CharacterTokenizer(vocab=_A		,   unk_token='[UNK]'	)

							self.assertListEqual(tokenizer.tokenize(''	)		,   []	)

							self.assertListEqual(tokenizer.tokenize('こんにちは'	)		,   ['こ', 'ん', 'に', 'ち', 'は']	)

							self.assertListEqual(tokenizer.tokenize('こんにちほ'	)		,   ['こ', 'ん', 'に', 'ち', '[UNK]']	)






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    str:







							"""simple docstring"""


							snake_case_		:			str											=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'	)

							snake_case_		:			Optional[Any]											=	tokenizer.encode('ありがとう。'		,   add_special_tokens=_A	)
							snake_case_		:			List[str]											=	tokenizer.encode('どういたしまして。'		,   add_special_tokens=_A	)

							snake_case_		:			List[Any]											=	tokenizer.build_inputs_with_special_tokens(_A	)
							snake_case_		:			Tuple											=	tokenizer.build_inputs_with_special_tokens(_A		,   _A	)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]
@custom_tokenizers
class SCREAMING_SNAKE_CASE_						(					unittest.TestCase					):






						def UpperCAmelCase_				(      self      :      str	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	'cl-tohoku/bert-base-japanese'
							snake_case_		:			List[Any]											=	AutoTokenizer.from_pretrained(_A	)
							self.assertIsInstance(_A		,   _A	)
class SCREAMING_SNAKE_CASE_						(					unittest.TestCase					):






						def UpperCAmelCase_				(      self      :      int	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	'cl-tohoku/bert-base-japanese'
							with self.assertLogs('transformers'		,   level='WARNING'	) as cm:
								BertTokenizer.from_pretrained(_A	)
								self.assertTrue(
								    cm.records[0].message.startswith(
								        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
								        ' is called from.'	)	)
							snake_case_		:			int											=	'bert-base-cased'
							with self.assertLogs('transformers'		,   level='WARNING'	) as cm:
								BertJapaneseTokenizer.from_pretrained(_A	)
								self.assertTrue(
								    cm.records[0].message.startswith(
								        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
								        ' is called from.'	)	)

