




'''simple docstring'''


import inspect
import unittest

from transformers import YolosConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import YolosForObjectDetection, YolosModel
 from transformers.models.yolos.modeling_yolos import YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor







class      SCREAMING_SNAKE_CASE    :






      def __init__( self   :  Any						,						lowercase__   :  int						,						lowercase__   :  int=13						,						lowercase__   :  Any=[30, 30]						,						lowercase__   :  List[str]=2						,						lowercase__   :  str=3						,						lowercase__   :  Dict=True						,						lowercase__   :  Union[str, Any]=True						,						lowercase__   :  Dict=32						,						lowercase__   :  List[str]=5						,						lowercase__   :  Optional[Any]=4						,						lowercase__   :  Tuple=37						,						lowercase__   :  Dict="gelu"						,						lowercase__   :  Optional[Any]=0.1						,						lowercase__   :  Tuple=0.1						,						lowercase__   :  str=10						,						lowercase__   :  Any=0.02						,						lowercase__   :  Any=3						,						lowercase__   :  Dict=None						,						lowercase__   :  Optional[int]=8						,						lowercase__   :  Union[str, Any]=10						,						):


           '''simple docstring'''

           a_     :							Tuple     						=  parent
           a_     :							Union[str, Any]     						=  batch_size
           a_     :							Optional[int]     						=  image_size
           a_     :							Optional[Any]     						=  patch_size
           a_     :							List[Any]     						=  num_channels
           a_     :							Union[str, Any]     						=  is_training
           a_     :							Optional[Any]     						=  use_labels
           a_     :							List[str]     						=  hidden_size
           a_     :							Dict     						=  num_hidden_layers
           a_     :							int     						=  num_attention_heads
           a_     :							Tuple     						=  intermediate_size
           a_     :							Optional[Any]     						=  hidden_act
           a_     :							Any     						=  hidden_dropout_prob
           a_     :							List[str]     						=  attention_probs_dropout_prob
           a_     :							str     						=  type_sequence_label_size
           a_     :							Dict     						=  initializer_range
           a_     :							Union[str, Any]     						=  num_labels
           a_     :							Union[str, Any]     						=  scope
           a_     :							str     						=  n_targets
           a_     :							Optional[Any]     						=  num_detection_tokens
           # we set the expected sequence length (which is used in several tests)
           # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + num_detection_tokens
           a_     :							Tuple     						=  (image_size[1] // patch_size) * (image_size[0] // patch_size)
           a_     :							str     						=  num_patches + 1 + self.num_detection_tokens






      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           a_     :							List[str]     						=  floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]       )

           a_     :							Union[str, Any]     						=  None
           if self.use_labels:
                # labels is a list of Dict (each Dict being the labels for a given example in the batch)
                a_     :							Optional[int]     						=  []
                for i in range(self.batch_size       ):
                     a_     :							Tuple     						=  {}
                     a_     :							Union[str, Any]     						=  torch.randint(
                         high=self.num_labels						,						size=(self.n_targets,)						,						device=lowercase__       )
                     a_     :							Optional[Any]     						=  torch.rand(self.n_targets						,						4						,						device=lowercase__       )
                     labels.append(lowercase__       )

           a_     :							Union[str, Any]     						=  self.get_config()

           return config, pixel_values, labels






      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           return YolosConfig(
               image_size=self.image_size						,						patch_size=self.patch_size						,						num_channels=self.num_channels						,						hidden_size=self.hidden_size						,						num_hidden_layers=self.num_hidden_layers						,						num_attention_heads=self.num_attention_heads						,						intermediate_size=self.intermediate_size						,						hidden_act=self.hidden_act						,						hidden_dropout_prob=self.hidden_dropout_prob						,						attention_probs_dropout_prob=self.attention_probs_dropout_prob						,						is_decoder=lowercase__						,						initializer_range=self.initializer_range						,						num_detection_tokens=self.num_detection_tokens						,						num_labels=self.num_labels						,						)






      def 							lowercase_						( self   :  str						,						lowercase__   :  int						,						lowercase__   :  Dict						,						lowercase__   :  Any       ):


           '''simple docstring'''

           a_     :							str     						=  YolosModel(config=lowercase__       )
           model.to(lowercase__       )
           model.eval()
           a_     :							Optional[int]     						=  model(lowercase__       )
           self.parent.assertEqual(
               result.last_hidden_state.shape						,						(self.batch_size, self.expected_seq_len, self.hidden_size)       )






      def 							lowercase_						( self   :  Dict						,						lowercase__   :  List[Any]						,						lowercase__   :  List[Any]						,						lowercase__   :  Optional[int]       ):


           '''simple docstring'''

           a_     :							Dict     						=  YolosForObjectDetection(lowercase__       )
           model.to(lowercase__       )
           model.eval()

           a_     :							str     						=  model(pixel_values=lowercase__       )
           a_     :							Optional[int]     						=  model(lowercase__       )

           self.parent.assertEqual(result.logits.shape						,						(self.batch_size, self.num_detection_tokens, self.num_labels + 1)       )
           self.parent.assertEqual(result.pred_boxes.shape						,						(self.batch_size, self.num_detection_tokens, 4)       )

           a_     :							Dict     						=  model(pixel_values=lowercase__						,						labels=lowercase__       )

           self.parent.assertEqual(result.loss.shape						,						()       )
           self.parent.assertEqual(result.logits.shape						,						(self.batch_size, self.num_detection_tokens, self.num_labels + 1)       )
           self.parent.assertEqual(result.pred_boxes.shape						,						(self.batch_size, self.num_detection_tokens, 4)       )






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  self.prepare_config_and_inputs()
           a_						,     a_						,     a_     :							List[Any]     						=  config_and_inputs
           a_     :							str     						=  {"""pixel_values""": pixel_values}
           return config, inputs_dict







@require_torch
class      SCREAMING_SNAKE_CASE    (						snake_case_							,      snake_case_							,      unittest.TestCase				):
      __magic_name__  :       Optional[Any]   							=					(YolosModel, YolosForObjectDetection) if is_torch_available() else ()
      __magic_name__  :       Any   							=					(
          {'''feature-extraction''': YolosModel, '''object-detection''': YolosForObjectDetection} if is_torch_available() else {}
      )

      __magic_name__  :       Optional[Any]   							=					False
      __magic_name__  :       Dict   							=					False
      __magic_name__  :       str   							=					False
      __magic_name__  :       Tuple   							=					False






      def 							lowercase_						( self   :  Any						,						lowercase__   :  int						,						lowercase__   :  Tuple						,						lowercase__   :  Union[str, Any]=False       ):


           '''simple docstring'''

           a_     :							str     						=  super()._prepare_for_class(lowercase__						,						lowercase__						,						return_labels=lowercase__       )

           if return_labels:
                if model_class.__name__ == "YolosForObjectDetection":
                     a_     :							int     						=  []
                     for i in range(self.model_tester.batch_size       ):
                          a_     :							Tuple     						=  {}
                          a_     :							Any     						=  torch.ones(
                              size=(self.model_tester.n_targets,)						,						device=lowercase__						,						dtype=torch.long       )
                          a_     :							Optional[Any]     						=  torch.ones(
                              self.model_tester.n_targets						,						4						,						device=lowercase__						,						dtype=torch.float       )
                          labels.append(lowercase__       )
                     a_     :							List[str]     						=  labels

           return inputs_dict






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  YolosModelTester(self       )
           a_     :							Tuple     						=  ConfigTester(self						,						config_class=lowercase__						,						has_text_modality=lowercase__						,						hidden_size=37       )






      def 							lowercase_						( self   :  Union[str, Any]       ):


           '''simple docstring'''

           self.config_tester.run_common_tests()






      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           pass






      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_						,     a_     :							Union[str, Any]     						=  self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                a_     :							Any     						=  model_class(lowercase__       )
                self.assertIsInstance(model.get_input_embeddings()						,						(nn.Module)       )
                a_     :							Optional[int]     						=  model.get_output_embeddings()
                self.assertTrue(x is None or isinstance(lowercase__						,						nn.Linear       )       )






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_						,     a_     :							Dict     						=  self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                a_     :							Dict     						=  model_class(lowercase__       )
                a_     :							int     						=  inspect.signature(model.forward       )
                # signature.parameters is an OrderedDict => so arg_names order is deterministic
                a_     :							Optional[Any]     						=  [*signature.parameters.keys()]

                a_     :							List[str]     						=  ["""pixel_values"""]
                self.assertListEqual(arg_names[:1]						,						lowercase__       )






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*lowercase__       )






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           a_						,     a_     :							Tuple     						=  self.model_tester.prepare_config_and_inputs_for_common()
           a_     :							List[Any]     						=  True

           # in YOLOS, the seq_len is different
           a_     :							str     						=  self.model_tester.expected_seq_len
           for model_class in self.all_model_classes:
                a_     :							str     						=  True
                a_     :							Optional[Any]     						=  False
                a_     :							Optional[int]     						=  True
                a_     :							Union[str, Any]     						=  model_class(lowercase__       )
                model.to(lowercase__       )
                model.eval()
                with torch.no_grad():
                     a_     :							Union[str, Any]     						=  model(**self._prepare_for_class(lowercase__						,						lowercase__       )       )
                a_     :							List[Any]     						=  outputs.attentions
                self.assertEqual(len(lowercase__       )						,						self.model_tester.num_hidden_layers       )

                # check that output_attentions also work using config
                del inputs_dict["output_attentions"]
                a_     :							int     						=  True
                a_     :							List[Any]     						=  model_class(lowercase__       )
                model.to(lowercase__       )
                model.eval()
                with torch.no_grad():
                     a_     :							int     						=  model(**self._prepare_for_class(lowercase__						,						lowercase__       )       )
                a_     :							Optional[int]     						=  outputs.attentions
                self.assertEqual(len(lowercase__       )						,						self.model_tester.num_hidden_layers       )

                self.assertListEqual(
                    list(attentions[0].shape[-3:]       )						,						[self.model_tester.num_attention_heads, seq_len, seq_len]						,						)
                a_     :							int     						=  len(lowercase__       )

                # Check attention is always last and order is fine
                a_     :							int     						=  True
                a_     :							List[Any]     						=  True
                a_     :							Optional[Any]     						=  model_class(lowercase__       )
                model.to(lowercase__       )
                model.eval()
                with torch.no_grad():
                     a_     :							List[Any]     						=  model(**self._prepare_for_class(lowercase__						,						lowercase__       )       )

                a_     :							Optional[Any]     						=  1
                self.assertEqual(out_len + added_hidden_states						,						len(lowercase__       )       )

                a_     :							int     						=  outputs.attentions

                self.assertEqual(len(lowercase__       )						,						self.model_tester.num_hidden_layers       )
                self.assertListEqual(
                    list(self_attentions[0].shape[-3:]       )						,						[self.model_tester.num_attention_heads, seq_len, seq_len]						,						)






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           def check_hidden_states_output(lowercase__   :  str						,						lowercase__   :  str						,						lowercase__   :  Dict       ):
                a_     :							Tuple     						=  model_class(lowercase__       )
                model.to(lowercase__       )
                model.eval()

                with torch.no_grad():
                     a_     :							Dict     						=  model(**self._prepare_for_class(lowercase__						,						lowercase__       )       )

                a_     :							List[Any]     						=  outputs.hidden_states

                a_     :							List[str]     						=  getattr(
                    self.model_tester						,						"""expected_num_hidden_layers"""						,						self.model_tester.num_hidden_layers + 1       )
                self.assertEqual(len(lowercase__       )						,						lowercase__       )

                # YOLOS has a different seq_length
                a_     :							List[Any]     						=  self.model_tester.expected_seq_len

                self.assertListEqual(
                    list(hidden_states[0].shape[-2:]       )						,						[seq_length, self.model_tester.hidden_size]						,						)

           a_						,     a_     :							Dict     						=  self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                a_     :							int     						=  True
                check_hidden_states_output(lowercase__						,						lowercase__						,						lowercase__       )

                # check that output_hidden_states also work using config
                del inputs_dict["output_hidden_states"]
                a_     :							str     						=  True

                check_hidden_states_output(lowercase__						,						lowercase__						,						lowercase__       )






      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           a_     :							Dict     						=  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_object_detection(*lowercase__       )






      @slow
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           for model_name in YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                a_     :							Optional[int]     						=  YolosModel.from_pretrained(lowercase__       )
                self.assertIsNotNone(lowercase__       )



def  _SCREAMING_SNAKE_CASE     (      ):




     """simple docstring"""


     a_     :							Optional[int]     						=  Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""				)
     return image







@require_torch
@require_vision
class      SCREAMING_SNAKE_CASE    (						unittest.TestCase				):






      @cached_property
      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           return AutoImageProcessor.from_pretrained("""hustvl/yolos-small"""       ) if is_vision_available() else None






      @slow
      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  YolosForObjectDetection.from_pretrained("""hustvl/yolos-small"""       ).to(lowercase__       )

           a_     :							Tuple     						=  self.default_image_processor
           a_     :							Dict     						=  prepare_img()
           a_     :							str     						=  image_processor(images=lowercase__						,						return_tensors="""pt"""       ).to(lowercase__       )

           # forward pass
           with torch.no_grad():
                a_     :							List[Any]     						=  model(inputs.pixel_values       )

           # verify outputs
           a_     :							Optional[int]     						=  torch.Size((1, 100, 92)       )
           self.assertEqual(outputs.logits.shape						,						lowercase__       )

           a_     :							Any     						=  torch.tensor(
               [[-24.0248, -10.3024, -14.8290], [-42.0392, -16.8200, -27.4334], [-27.2743, -11.8154, -18.7148]]						,						device=lowercase__						,						)
           a_     :							str     						=  torch.tensor(
               [[0.2559, 0.5455, 0.4706], [0.2989, 0.7279, 0.1875], [0.7732, 0.4017, 0.4462]]						,						device=lowercase__       )
           self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]						,						lowercase__						,						atol=1e-4       )       )
           self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3]						,						lowercase__						,						atol=1e-4       )       )

           # verify postprocessing
           a_     :							Any     						=  image_processor.post_process_object_detection(
               lowercase__						,						threshold=0.3						,						target_sizes=[image.size[::-1]]       )[0]
           a_     :							Optional[Any]     						=  torch.tensor([0.9994, 0.9790, 0.9964, 0.9972, 0.9861]       ).to(lowercase__       )
           a_     :							Optional[Any]     						=  [75, 75, 17, 63, 17]
           a_     :							Optional[Any]     						=  torch.tensor([335.0609, 79.3848, 375.4216, 187.2495]       ).to(lowercase__       )

           self.assertEqual(len(results["""scores"""]       )						,						5       )
           self.assertTrue(torch.allclose(results["""scores"""]						,						lowercase__						,						atol=1e-4       )       )
           self.assertSequenceEqual(results["""labels"""].tolist()						,						lowercase__       )
           self.assertTrue(torch.allclose(results["""boxes"""][0, :]						,						lowercase__       )       )






'''simple docstring'''


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class      SCREAMING_SNAKE_CASE    (						snake_case_							,      unittest.TestCase				):
      __magic_name__  :       Optional[Any]   							=					BertJapaneseTokenizer
      __magic_name__  :       int   							=					False
      __magic_name__  :       Tuple   							=					True






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           super().setUp()

           a_     :							List[Any]     						=  [
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """こんにちは""",
               """こん""",
               """にちは""",
               """ばんは""",
               """##こん""",
               """##にちは""",
               """##ばんは""",
               """世界""",
               """##世界""",
               """、""",
               """##、""",
               """。""",
               """##。""",
           ]

           a_     :							Optional[int]     						=  os.path.join(self.tmpdirname						,						VOCAB_FILES_NAMES["""vocab_file"""]       )
           with open(self.vocab_file						,						"""w"""						,						encoding="""utf-8"""       ) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






      def 							lowercase_						( self   :  List[Any]						,						lowercase__   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Dict     						=  """こんにちは、世界。 \nこんばんは、世界。"""
           a_     :							Tuple     						=  """こんにちは 、 世界 。 こんばんは 、 世界 。"""
           return input_text, output_text






      def 							lowercase_						( self   :  Optional[int]						,						lowercase__   :  str       ):


           '''simple docstring'''

           a_						,     a_     :							int     						=  self.get_input_output_texts(lowercase__       )
           a_     :							Tuple     						=  tokenizer.encode(lowercase__						,						add_special_tokens=lowercase__       )
           a_     :							Union[str, Any]     						=  tokenizer.decode(lowercase__						,						clean_up_tokenization_spaces=lowercase__       )
           return text, ids






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Tuple     						=  self.tokenizer_class(self.vocab_file       )

           a_     :							Union[str, Any]     						=  tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )






      def 							lowercase_						( self   :  Union[str, Any]       ):


           '''simple docstring'''

           a_     :							Tuple     						=  self.tokenizer_class(self.vocab_file						,						word_tokenizer_type="""mecab"""       )
           self.assertIsNotNone(lowercase__       )

           a_     :							Optional[Any]     						=  """こんにちは、世界。\nこんばんは、世界。"""
           a_     :							Any     						=  tokenizer.tokenize(lowercase__       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

           a_     :							Dict     						=  os.path.join(self.tmpdirname						,						"""tokenizer.bin"""       )
           with open(lowercase__						,						"""wb"""       ) as handle:
                pickle.dump(lowercase__						,						lowercase__       )

           with open(lowercase__						,						"""rb"""       ) as handle:
                a_     :							Optional[int]     						=  pickle.load(lowercase__       )

           a_     :							Tuple     						=  tokenizer_new.tokenize(lowercase__       )

           self.assertListEqual(lowercase__						,						lowercase__       )






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							List[str]     						=  MecabTokenizer(mecab_dic="""ipadic"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           try:
                a_     :							Tuple     						=  MecabTokenizer(mecab_dic="""unidic_lite"""       )
           except ModuleNotFoundError:
                return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           try:
                a_     :							Union[str, Any]     						=  MecabTokenizer(mecab_dic="""unidic"""       )
           except ModuleNotFoundError:
                return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  MecabTokenizer(do_lower_case=lowercase__						,						mecab_dic="""ipadic"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           try:
                a_     :							Any     						=  MecabTokenizer(
                    do_lower_case=lowercase__						,						normalize_text=lowercase__						,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""       )
           except RuntimeError:
                # if dict doesn't exist in the system, previous code raises this error.
                return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							str     						=  MecabTokenizer(normalize_text=lowercase__						,						mecab_dic="""ipadic"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							int     						=  self.tokenizer_class(self.vocab_file						,						word_tokenizer_type="""sudachi"""       )
           self.assertIsNotNone(lowercase__       )

           a_     :							Union[str, Any]     						=  """こんにちは、世界。\nこんばんは、世界。"""
           a_     :							Optional[int]     						=  tokenizer.tokenize(lowercase__       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

           a_     :							str     						=  os.path.join(self.tmpdirname						,						"""tokenizer.bin"""       )
           with open(lowercase__						,						"""wb"""       ) as handle:
                pickle.dump(lowercase__						,						lowercase__       )

           with open(lowercase__						,						"""rb"""       ) as handle:
                a_     :							List[Any]     						=  pickle.load(lowercase__       )

           a_     :							Optional[int]     						=  tokenizer_new.tokenize(lowercase__       )

           self.assertListEqual(lowercase__						,						lowercase__       )






      @require_sudachi
      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  SudachiTokenizer(sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  SudachiTokenizer(sudachi_dict_type="""core"""						,						sudachi_split_mode="""A"""       )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )						,						["""外国""", """人""", """参政""", """権"""]       )






      @require_sudachi
      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Dict     						=  SudachiTokenizer(sudachi_dict_type="""core"""						,						sudachi_split_mode="""B"""       )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )						,						["""外国人""", """参政権"""]       )






      @require_sudachi
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Tuple     						=  SudachiTokenizer(sudachi_dict_type="""core"""						,						sudachi_split_mode="""C"""       )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )						,						["""外国人参政権"""]       )






      @require_sudachi
      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							List[Any]     						=  SudachiTokenizer(do_lower_case=lowercase__						,						sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Tuple     						=  SudachiTokenizer(normalize_text=lowercase__						,						sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  SudachiTokenizer(trim_whitespace=lowercase__						,						sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  self.tokenizer_class(self.vocab_file						,						word_tokenizer_type="""jumanpp"""       )
           self.assertIsNotNone(lowercase__       )

           a_     :							Union[str, Any]     						=  """こんにちは、世界。\nこんばんは、世界。"""
           a_     :							Union[str, Any]     						=  tokenizer.tokenize(lowercase__       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

           a_     :							Optional[Any]     						=  os.path.join(self.tmpdirname						,						"""tokenizer.bin"""       )
           with open(lowercase__						,						"""wb"""       ) as handle:
                pickle.dump(lowercase__						,						lowercase__       )

           with open(lowercase__						,						"""rb"""       ) as handle:
                a_     :							Any     						=  pickle.load(lowercase__       )

           a_     :							List[Any]     						=  tokenizer_new.tokenize(lowercase__       )

           self.assertListEqual(lowercase__						,						lowercase__       )






      @require_jumanpp
      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							int     						=  JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							str     						=  JumanppTokenizer(do_lower_case=lowercase__       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           a_     :							List[Any]     						=  JumanppTokenizer(normalize_text=lowercase__       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  JumanppTokenizer(trim_whitespace=lowercase__       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""       )						,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]						,						)






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							int     						=  ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

           a_     :							int     						=  {}
           for i, token in enumerate(lowercase__       ):
                a_     :							str     						=  i
           a_     :							int     						=  WordpieceTokenizer(vocab=lowercase__						,						unk_token="""[UNK]"""       )

           self.assertListEqual(tokenizer.tokenize(""""""       )						,						[]       )

           self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )						,						["""こんにちは"""]       )

           self.assertListEqual(tokenizer.tokenize("""こんばんは"""       )						,						["""こん""", """##ばんは"""]       )

           self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""       )						,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]       )






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""       )
           a_     :							List[str]     						=  tokenizer.subword_tokenizer

           a_     :							Any     						=  subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""       )
           self.assertListEqual(lowercase__						,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]       )

           a_     :							Optional[Any]     						=  subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""       )
           self.assertListEqual(lowercase__						,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]       )






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""       )

           a_     :							Union[str, Any]     						=  tokenizer.encode("""ありがとう。"""						,						add_special_tokens=lowercase__       )
           a_     :							Any     						=  tokenizer.encode("""どういたしまして。"""						,						add_special_tokens=lowercase__       )

           a_     :							str     						=  tokenizer.build_inputs_with_special_tokens(lowercase__       )
           a_     :							Optional[int]     						=  tokenizer.build_inputs_with_special_tokens(lowercase__						,						lowercase__       )

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      SCREAMING_SNAKE_CASE    (						snake_case_							,      unittest.TestCase				):
      __magic_name__  :       Optional[Any]   							=					BertJapaneseTokenizer
      __magic_name__  :       int   							=					False






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           super().setUp()

           a_     :							int     						=  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

           a_     :							Tuple     						=  os.path.join(self.tmpdirname						,						VOCAB_FILES_NAMES["""vocab_file"""]       )
           with open(self.vocab_file						,						"""w"""						,						encoding="""utf-8"""       ) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






      def 							lowercase_						( self   :  str						,						**lowercase__   :  Tuple       ):


           '''simple docstring'''

           return BertJapaneseTokenizer.from_pretrained(self.tmpdirname						,						subword_tokenizer_type="""character"""						,						**lowercase__       )






      def 							lowercase_						( self   :  str						,						lowercase__   :  Union[str, Any]       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  """こんにちは、世界。 \nこんばんは、世界。"""
           a_     :							List[Any]     						=  """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
           return input_text, output_text






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  self.tokenizer_class(self.vocab_file						,						subword_tokenizer_type="""character"""       )

           a_     :							Tuple     						=  tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""       )
           self.assertListEqual(
               lowercase__						,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]       )
           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]       )






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							str     						=  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

           a_     :							Dict     						=  {}
           for i, token in enumerate(lowercase__       ):
                a_     :							List[Any]     						=  i
           a_     :							Dict     						=  CharacterTokenizer(vocab=lowercase__						,						unk_token="""[UNK]"""       )

           self.assertListEqual(tokenizer.tokenize(""""""       )						,						[]       )

           self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )						,						["""こ""", """ん""", """に""", """ち""", """は"""]       )

           self.assertListEqual(tokenizer.tokenize("""こんにちほ"""       )						,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""]       )






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           a_     :							Dict     						=  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""       )

           a_     :							List[Any]     						=  tokenizer.encode("""ありがとう。"""						,						add_special_tokens=lowercase__       )
           a_     :							List[Any]     						=  tokenizer.encode("""どういたしまして。"""						,						add_special_tokens=lowercase__       )

           a_     :							str     						=  tokenizer.build_inputs_with_special_tokens(lowercase__       )
           a_     :							Optional[Any]     						=  tokenizer.build_inputs_with_special_tokens(lowercase__						,						lowercase__       )

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      SCREAMING_SNAKE_CASE    (						unittest.TestCase				):






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  """cl-tohoku/bert-base-japanese"""
           a_     :							Union[str, Any]     						=  AutoTokenizer.from_pretrained(lowercase__       )
           self.assertIsInstance(lowercase__						,						lowercase__       )







class      SCREAMING_SNAKE_CASE    (						unittest.TestCase				):






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Any     						=  """cl-tohoku/bert-base-japanese"""
           with self.assertLogs("""transformers"""						,						level="""WARNING"""       ) as cm:
                BertTokenizer.from_pretrained(lowercase__       )
                self.assertTrue(
                    cm.records[0].message.startswith(
                        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                        """ is called from."""       )       )
           a_     :							Optional[Any]     						=  """bert-base-cased"""
           with self.assertLogs("""transformers"""						,						level="""WARNING"""       ) as cm:
                BertJapaneseTokenizer.from_pretrained(lowercase__       )
                self.assertTrue(
                    cm.records[0].message.startswith(
                        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                        """ is called from."""       )       )
