





'''simple docstring'''






import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    XLMForMultipleChoice,
	    XLMForQuestionAnswering,
	    XLMForQuestionAnsweringSimple,
	    XLMForSequenceClassification,
	    XLMForTokenClassification,
	    XLMModel,
	    XLMWithLMHeadModel,
	)
	from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST




class  _A :

		def __init__(   self					:   List[Any]  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   Optional[int]=13  ,      __magic_name__					:   Any=7  ,      __magic_name__					:   Any=True  ,      __magic_name__					:   Optional[int]=True  ,      __magic_name__					:   Any=True  ,      __magic_name__					:   str=True  ,      __magic_name__					:   Tuple=True  ,      __magic_name__					:   str=False  ,      __magic_name__					:   List[Any]=False  ,      __magic_name__					:   Optional[Any]=False  ,      __magic_name__					:   int=2  ,      __magic_name__					:   Any=99  ,      __magic_name__					:   Union[str, Any]=0  ,      __magic_name__					:   Tuple=32  ,      __magic_name__					:   Optional[int]=5  ,      __magic_name__					:   Tuple=4  ,      __magic_name__					:   int=0.1  ,      __magic_name__					:   Optional[Any]=0.1  ,      __magic_name__					:   int=5_12  ,      __magic_name__					:   str=2  ,      __magic_name__					:   Tuple=0.02  ,      __magic_name__					:   List[str]=2  ,      __magic_name__					:   List[Any]=4  ,      __magic_name__					:   str="last"  ,      __magic_name__					:   Dict=True  ,      __magic_name__					:   Optional[int]=None  ,      __magic_name__					:   int=0  ,      )					->    str:


					"""simple docstring"""

					__snake_case					:      List[str]							      =     parent
					__snake_case					:      List[str]							      =     batch_size
					__snake_case					:      List[str]							      =     seq_length
					__snake_case					:      Dict							      =     is_training
					__snake_case					:      Optional[Any]							      =     use_input_lengths
					__snake_case					:      Optional[int]							      =     use_token_type_ids
					__snake_case					:      int							      =     use_labels
					__snake_case					:      Any							      =     gelu_activation
					__snake_case					:      List[str]							      =     sinusoidal_embeddings
					__snake_case					:      int							      =     causal
					__snake_case					:      int							      =     asm
					__snake_case					:      Any							      =     n_langs
					__snake_case					:      Any							      =     vocab_size
					__snake_case					:      str							      =     n_special
					__snake_case					:      Union[str, Any]							      =     hidden_size
					__snake_case					:      Optional[Any]							      =     num_hidden_layers
					__snake_case					:      int							      =     num_attention_heads
					__snake_case					:      Any							      =     hidden_dropout_prob
					__snake_case					:      Any							      =     attention_probs_dropout_prob
					__snake_case					:      Optional[int]							      =     max_position_embeddings
					__snake_case					:      str							      =     type_sequence_label_size
					__snake_case					:      Optional[int]							      =     initializer_range
					__snake_case					:      List[str]							      =     num_labels
					__snake_case					:      List[str]							      =     num_choices
					__snake_case					:      Tuple							      =     summary_type
					__snake_case					:      List[str]							      =     use_proj
					__snake_case					:      Dict							      =     scope
					__snake_case					:      Any							      =     bos_token_id

		def 	lowercase__     (   self					:   List[str]			)					->    Dict:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     ids_tensor([self.batch_size, self.seq_length]  ,      self.vocab_size			)
					__snake_case					:      List[Any]							      =     random_attention_mask([self.batch_size, self.seq_length]			)

					__snake_case					:      List[Any]							      =     None
					if self.use_input_lengths:
								__snake_case					:      Optional[int]							      =     (
								    ids_tensor([self.batch_size]  ,      vocab_size=2			) + self.seq_length - 2
								)  # small variation of seq_length

					__snake_case					:      Optional[Any]							      =     None
					if self.use_token_type_ids:
								__snake_case					:      int							      =     ids_tensor([self.batch_size, self.seq_length]  ,      self.n_langs			)

					__snake_case					:      List[str]							      =     None
					__snake_case					:      List[Any]							      =     None
					__snake_case					:      List[Any]							      =     None
					if self.use_labels:
								__snake_case					:      Optional[int]							      =     ids_tensor([self.batch_size]  ,      self.type_sequence_label_size			)
								__snake_case					:      Optional[int]							      =     ids_tensor([self.batch_size, self.seq_length]  ,      self.num_labels			)
								__snake_case					:      int							      =     ids_tensor([self.batch_size]  ,      2			).float()
								__snake_case					:      List[str]							      =     ids_tensor([self.batch_size]  ,      self.num_choices			)

					__snake_case					:      Any							      =     self.get_config()

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_lengths,
					    sequence_labels,
					    token_labels,
					    is_impossible_labels,
					    choice_labels,
					    input_mask,
					)

		def 	lowercase__     (   self					:   str			)					->    Optional[Any]:


					"""simple docstring"""

					return XLMConfig(
					    vocab_size=self.vocab_size  ,      n_special=self.n_special  ,      emb_dim=self.hidden_size  ,      n_layers=self.num_hidden_layers  ,      n_heads=self.num_attention_heads  ,      dropout=self.hidden_dropout_prob  ,      attention_dropout=self.attention_probs_dropout_prob  ,      gelu_activation=self.gelu_activation  ,      sinusoidal_embeddings=self.sinusoidal_embeddings  ,      asm=self.asm  ,      causal=self.causal  ,      n_langs=self.n_langs  ,      max_position_embeddings=self.max_position_embeddings  ,      initializer_range=self.initializer_range  ,      summary_type=self.summary_type  ,      use_proj=self.use_proj  ,      num_labels=self.num_labels  ,      bos_token_id=self.bos_token_id  ,      )

		def 	lowercase__     (   self					:   List[str]  ,      __magic_name__					:   str  ,      __magic_name__					:   Dict  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   Any  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   Union[str, Any]  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Any  ,      )					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     XLMModel(config=__magic_name__			)
					model.to(__magic_name__			)
					model.eval()
					__snake_case					:      Any							      =     model(__magic_name__  ,      lengths=__magic_name__  ,      langs=__magic_name__			)
					__snake_case					:      List[str]							      =     model(__magic_name__  ,      langs=__magic_name__			)
					__snake_case					:      Optional[int]							      =     model(__magic_name__			)
					self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)			)

		def 	lowercase__     (   self					:   Tuple  ,      __magic_name__					:   Optional[Any]  ,      __magic_name__					:   str  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   int  ,      __magic_name__					:   str  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   str  ,      __magic_name__					:   Optional[int]  ,      )					->    List[str]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     XLMWithLMHeadModel(__magic_name__			)
					model.to(__magic_name__			)
					model.eval()

					__snake_case					:      Any							      =     model(__magic_name__  ,      token_type_ids=__magic_name__  ,      labels=__magic_name__			)
					self.parent.assertEqual(result.loss.shape  ,      ()			)
					self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.vocab_size)			)

		def 	lowercase__     (   self					:   Dict  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   Dict  ,      __magic_name__					:   int  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   Dict  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   Union[str, Any]  ,      )					->    List[str]:


					"""simple docstring"""

					__snake_case					:      int							      =     XLMForQuestionAnsweringSimple(__magic_name__			)
					model.to(__magic_name__			)
					model.eval()

					__snake_case					:      int							      =     model(__magic_name__			)

					__snake_case					:      Any							      =     model(__magic_name__  ,      start_positions=__magic_name__  ,      end_positions=__magic_name__			)
					__snake_case					:      Tuple							      =     outputs
					self.parent.assertEqual(result.start_logits.shape  ,      (self.batch_size, self.seq_length)			)
					self.parent.assertEqual(result.end_logits.shape  ,      (self.batch_size, self.seq_length)			)

		def 	lowercase__     (   self					:   int  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Dict  ,      __magic_name__					:   Union[str, Any]  ,      __magic_name__					:   Union[str, Any]  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   Tuple  ,      )					->    int:


					"""simple docstring"""

					__snake_case					:      Dict							      =     XLMForQuestionAnswering(__magic_name__			)
					model.to(__magic_name__			)
					model.eval()

					__snake_case					:      Optional[Any]							      =     model(__magic_name__			)

					__snake_case					:      Optional[Any]							      =     model(
					    __magic_name__  ,      start_positions=__magic_name__  ,      end_positions=__magic_name__  ,      cls_index=__magic_name__  ,      is_impossible=__magic_name__  ,      p_mask=__magic_name__  ,      )

					__snake_case					:      Dict							      =     model(
					    __magic_name__  ,      start_positions=__magic_name__  ,      end_positions=__magic_name__  ,      cls_index=__magic_name__  ,      is_impossible=__magic_name__  ,      )

					((__snake_case)      ,     )					:      Dict							      =     result_with_labels.to_tuple()

					__snake_case					:      str							      =     model(__magic_name__  ,      start_positions=__magic_name__  ,      end_positions=__magic_name__			)

					((__snake_case)      ,     )					:      List[Any]							      =     result_with_labels.to_tuple()

					self.parent.assertEqual(result_with_labels.loss.shape  ,      ()			)
					self.parent.assertEqual(result.start_top_log_probs.shape  ,      (self.batch_size, model.config.start_n_top)			)
					self.parent.assertEqual(result.start_top_index.shape  ,      (self.batch_size, model.config.start_n_top)			)
					self.parent.assertEqual(
					    result.end_top_log_probs.shape  ,      (self.batch_size, model.config.start_n_top * model.config.end_n_top)			)
					self.parent.assertEqual(
					    result.end_top_index.shape  ,      (self.batch_size, model.config.start_n_top * model.config.end_n_top)			)
					self.parent.assertEqual(result.cls_logits.shape  ,      (self.batch_size,)			)

		def 	lowercase__     (   self					:   List[Any]  ,      __magic_name__					:   Union[str, Any]  ,      __magic_name__					:   str  ,      __magic_name__					:   Dict  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   int  ,      __magic_name__					:   Dict  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Tuple  ,      )					->    int:


					"""simple docstring"""

					__snake_case					:      List[str]							      =     XLMForSequenceClassification(__magic_name__			)
					model.to(__magic_name__			)
					model.eval()

					__snake_case					:      Optional[int]							      =     model(__magic_name__			)
					__snake_case					:      Optional[int]							      =     model(__magic_name__  ,      labels=__magic_name__			)
					self.parent.assertEqual(result.loss.shape  ,      ()			)
					self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.type_sequence_label_size)			)

		def 	lowercase__     (   self					:   Union[str, Any]  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   Any  ,      __magic_name__					:   Optional[Any]  ,      __magic_name__					:   int  ,      __magic_name__					:   Optional[Any]  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   Any  ,      __magic_name__					:   List[str]  ,      )					->    str:


					"""simple docstring"""

					__snake_case					:      Any							      =     self.num_labels
					__snake_case					:      Dict							      =     XLMForTokenClassification(__magic_name__			)
					model.to(__magic_name__			)
					model.eval()

					__snake_case					:      Optional[Any]							      =     model(__magic_name__  ,      attention_mask=__magic_name__  ,      labels=__magic_name__			)
					self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.num_labels)			)

		def 	lowercase__     (   self					:   Optional[Any]  ,      __magic_name__					:   int  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   int  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   int  ,      __magic_name__					:   Optional[Any]  ,      __magic_name__					:   Dict  ,      __magic_name__					:   List[str]  ,      )					->    Dict:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     self.num_choices
					__snake_case					:      Optional[Any]							      =     XLMForMultipleChoice(config=__magic_name__			)
					model.to(__magic_name__			)
					model.eval()
					__snake_case					:      List[str]							      =     input_ids.unsqueeze(1			).expand(-1  ,      self.num_choices  ,      -1			).contiguous()
					__snake_case					:      List[str]							      =     token_type_ids.unsqueeze(1			).expand(-1  ,      self.num_choices  ,      -1			).contiguous()
					__snake_case					:      List[str]							      =     input_mask.unsqueeze(1			).expand(-1  ,      self.num_choices  ,      -1			).contiguous()
					__snake_case					:      Optional[Any]							      =     model(
					    __magic_name__  ,      attention_mask=__magic_name__  ,      token_type_ids=__magic_name__  ,      labels=__magic_name__  ,      )
					self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.num_choices)			)

		def 	lowercase__     (   self					:   List[Any]			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      Optional[int]							      =     self.prepare_config_and_inputs()
					(
					    (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     (
					    __snake_case
					)      ,     
					)					:      Optional[Any]							      =     config_and_inputs
					__snake_case					:      Any							      =     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """lengths""": input_lengths}
					return config, inputs_dict





@require_torch
class  _A (					__lowercase						,					__lowercase						,					__lowercase						,					unittest.TestCase		):
		lowercase__:   Optional[Any]				     =   (
		    (
		        XLMModel,
		        XLMWithLMHeadModel,
		        XLMForQuestionAnswering,
		        XLMForSequenceClassification,
		        XLMForQuestionAnsweringSimple,
		        XLMForTokenClassification,
		        XLMForMultipleChoice,
		    )
		    if is_torch_available()
		    else ()
		)
		lowercase__:   Union[str, Any]				     =   (
		    (XLMWithLMHeadModel,) if is_torch_available() else ()
		)  # TODO (PVP): Check other models whether language generation is also applicable
		lowercase__:   Union[str, Any]				     =   (
		    {
		        '''feature-extraction''': XLMModel,
		        '''fill-mask''': XLMWithLMHeadModel,
		        '''question-answering''': XLMForQuestionAnsweringSimple,
		        '''text-classification''': XLMForSequenceClassification,
		        '''text-generation''': XLMWithLMHeadModel,
		        '''token-classification''': XLMForTokenClassification,
		        '''zero-shot''': XLMForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)

		def 	lowercase__     (   self					:   Optional[int]  ,      __magic_name__					:   List[Any]  ,      __magic_name__					:   str  ,      __magic_name__					:   Optional[int]  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   Optional[Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					if (
					    pipeline_test_casse_name == "QAPipelineTests"
					    and tokenizer_name is not None
					    and not tokenizer_name.endswith("""Fast"""			)
					):
								# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
								# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
								# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
								return True

					return False

		def 	lowercase__     (   self					:   Tuple  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   Union[str, Any]=False			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      List[str]							      =     super()._prepare_for_class(__magic_name__  ,      __magic_name__  ,      return_labels=__magic_name__			)

					if return_labels:
								if model_class.__name__ == "XLMForQuestionAnswering":
											__snake_case					:      List[Any]							      =     torch.zeros(
											    self.model_tester.batch_size  ,      dtype=torch.long  ,      device=__magic_name__			)
											__snake_case					:      Optional[Any]							      =     torch.zeros(
											    self.model_tester.batch_size  ,      dtype=torch.long  ,      device=__magic_name__			)

					return inputs_dict

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    str:


					"""simple docstring"""

					__snake_case					:      Any							      =     XLMModelTester(self			)
					__snake_case					:      List[str]							      =     ConfigTester(self  ,      config_class=__magic_name__  ,      emb_dim=37			)

		def 	lowercase__     (   self					:   int			)					->    Tuple:


					"""simple docstring"""

					self.config_tester.run_common_tests()

		def 	lowercase__     (   self					:   Dict			)					->    int:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_model(*__magic_name__			)

		def 	lowercase__     (   self					:   Any			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      Any							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_lm_head(*__magic_name__			)

		def 	lowercase__     (   self					:   Optional[int]			)					->    List[str]:


					"""simple docstring"""

					__snake_case					:      Any							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_simple_qa(*__magic_name__			)

		def 	lowercase__     (   self					:   List[str]			)					->    List[str]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_qa(*__magic_name__			)

		def 	lowercase__     (   self					:   Optional[int]			)					->    str:


					"""simple docstring"""

					__snake_case					:      Optional[Any]							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_sequence_classif(*__magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Dict:


					"""simple docstring"""

					__snake_case					:      List[str]							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_token_classif(*__magic_name__			)

		def 	lowercase__     (   self					:   Optional[Any]			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      Optional[int]							      =     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_for_multiple_choice(*__magic_name__			)

		def 	lowercase__     (   self					:   str  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   str  ,      __magic_name__					:   int  ,      __magic_name__					:   Any  ,      __magic_name__					:   int=False  ,      __magic_name__					:   Any=1			)					->    str:


					"""simple docstring"""

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)
					self.assertListEqual(
					    [isinstance(__magic_name__  ,      __magic_name__			) for iter_attentions in attentions]  ,      [True] * len(__magic_name__			)			)
					self.assertEqual(len(__magic_name__			)  ,      (max_length - min_length) * num_beam_groups			)

					for idx, iter_attentions in enumerate(__magic_name__			):
								# adds PAD dummy token
								__snake_case					:      List[str]							      =     min_length + idx + 1
								__snake_case					:      List[Any]							      =     min_length + idx + 1

								__snake_case					:      int							      =     (
								    batch_size * num_beam_groups,
								    config.num_attention_heads,
								    tgt_len,
								    src_len,
								)
								# check attn size
								self.assertListEqual(
								    [layer_attention.shape for layer_attention in iter_attentions]  ,      [expected_shape] * len(__magic_name__			)			)

		def 	lowercase__     (   self					:   Any  ,      __magic_name__					:   int  ,      __magic_name__					:   Dict  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Any  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   Dict=False  ,      __magic_name__					:   Optional[int]=1			)					->    Optional[int]:


					"""simple docstring"""

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)
					self.assertListEqual(
					    [isinstance(__magic_name__  ,      __magic_name__			) for iter_hidden_states in hidden_states]  ,      [True] * len(__magic_name__			)  ,      )
					self.assertEqual(len(__magic_name__			)  ,      (max_length - min_length) * num_beam_groups			)

					for idx, iter_hidden_states in enumerate(__magic_name__			):
								# adds PAD dummy token
								__snake_case					:      str							      =     min_length + idx + 1
								__snake_case					:      Union[str, Any]							      =     (batch_size * num_beam_groups, seq_len, config.hidden_size)
								# check hidden size
								self.assertListEqual(
								    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]  ,      [expected_shape] * len(__magic_name__			)  ,      )
					pass

		@slow
		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								__snake_case					:      Dict							      =     XLMModel.from_pretrained(__magic_name__			)
								self.assertIsNotNone(__magic_name__			)





@require_torch
class  _A (					unittest.TestCase		):

		@slow
		def 	lowercase__     (   self					:   str			)					->    Tuple:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     XLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048"""			)
					model.to(__magic_name__			)
					__snake_case					:      Any							      =     torch.tensor([[14, 4_47]]  ,      dtype=torch.long  ,      device=__magic_name__			)  # the president
					__snake_case					:      Union[str, Any]							      =     [
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					]  # the president the president the president the president the president the president the president the president the president the president
					# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
					__snake_case					:      Tuple							      =     model.generate(__magic_name__  ,      do_sample=__magic_name__			)
					self.assertListEqual(output_ids[0].cpu().numpy().tolist()  ,      __magic_name__			)





"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Union[str, Any]       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		str       		=				False
			lowerCamelCase__ :		Optional[int]       		=				True

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     str:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """こんにちは""",
						    """こん""",
						    """にちは""",
						    """ばんは""",
						    """##こん""",
						    """##にちは""",
						    """##ばんは""",
						    """世界""",
						    """##世界""",
						    """、""",
						    """##、""",
						    """。""",
						    """##。""",
						]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]     ,					__UpperCAmelCase					:    Optional[int]							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     int:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.get_input_output_texts(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.decode(__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase							)
						return text, ids

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Tuple:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""mecab"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Union[str, Any]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic_lite"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(do_lower_case=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     List[str]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(
									    do_lower_case=__UpperCAmelCase     ,					normalize_text=__UpperCAmelCase     ,					mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""							)
						except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(normalize_text=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""sudachi"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""A"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国""", """人""", """参政""", """権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""B"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人""", """参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""C"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(do_lower_case=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     str:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(normalize_text=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(trim_whitespace=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""jumanpp"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(do_lower_case=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(normalize_text=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(trim_whitespace=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""							)     ,					["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   WordpieceTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こんにちは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは"""							)     ,					["""こん""", """##ばんは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""							)     ,					["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.subword_tokenizer

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]							)

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Dict       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		Tuple       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]     ,					**__UpperCAmelCase					:    List[Any]							) ->     List[Any]:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     ,					subword_tokenizer_type="""character"""     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    Any							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Union[str, Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					subword_tokenizer_type="""character"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""							)
						self.assertListEqual(
						    __UpperCAmelCase     ,					["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]							)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   CharacterTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こ""", """ん""", """に""", """ち""", """は"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちほ"""							)     ,					["""こ""", """ん""", """に""", """ち""", """[UNK]"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     str:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						SCREAMING_SNAKE_CASE__										=   AutoTokenizer.from_pretrained(__UpperCAmelCase							)
						self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase							)





class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)
						SCREAMING_SNAKE_CASE__										=   """bert-base-cased"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertJapaneseTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)


