





'''simple docstring'''

import inspect
import unittest

from transformers import SegformerConfig, is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_MAPPING,
        SegformerForImageClassification,
        SegformerForSemanticSegmentation,
        SegformerModel,
    )
    from transformers.models.segformer.modeling_segformer import SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import SegformerImageProcessor







class __a			( _snake_case      ):





   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.config_class(**self.inputs_dict     )
      self.parent.assertTrue(hasattr(lowerCamelCase					,"""hidden_sizes"""     )     )
      self.parent.assertTrue(hasattr(lowerCamelCase					,"""num_attention_heads"""     )     )
      self.parent.assertTrue(hasattr(lowerCamelCase					,"""num_encoder_blocks"""     )     )







class __a			:





   def __init__(       self					:      Optional[Any]					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      List[Any]=13					,lowerCamelCase					:      Union[str, Any]=64					,lowerCamelCase					:      Dict=3					,lowerCamelCase					:      Optional[Any]=4					,lowerCamelCase					:      Optional[Any]=[2, 2, 2, 2]					,lowerCamelCase					:      Tuple=[8, 4, 2, 1]					,lowerCamelCase					:      Dict=[16, 32, 64, 128]					,lowerCamelCase					:      Tuple=[1, 4, 8, 16]					,lowerCamelCase					:      str=[1, 2, 4, 8]					,lowerCamelCase					:      str=True					,lowerCamelCase					:      Union[str, Any]=True					,lowerCamelCase					:      Optional[Any]="gelu"					,lowerCamelCase					:      Union[str, Any]=0.1					,lowerCamelCase					:      List[str]=0.1					,lowerCamelCase					:      Optional[Any]=0.02					,lowerCamelCase					:      int=3					,lowerCamelCase					:      List[str]=None					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							parent
      __SCREAMING_SNAKE_CASE   				=							batch_size
      __SCREAMING_SNAKE_CASE   				=							image_size
      __SCREAMING_SNAKE_CASE   				=							num_channels
      __SCREAMING_SNAKE_CASE   				=							num_encoder_blocks
      __SCREAMING_SNAKE_CASE   				=							sr_ratios
      __SCREAMING_SNAKE_CASE   				=							depths
      __SCREAMING_SNAKE_CASE   				=							hidden_sizes
      __SCREAMING_SNAKE_CASE   				=							downsampling_rates
      __SCREAMING_SNAKE_CASE   				=							num_attention_heads
      __SCREAMING_SNAKE_CASE   				=							is_training
      __SCREAMING_SNAKE_CASE   				=							use_labels
      __SCREAMING_SNAKE_CASE   				=							hidden_act
      __SCREAMING_SNAKE_CASE   				=							hidden_dropout_prob
      __SCREAMING_SNAKE_CASE   				=							attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE   				=							initializer_range
      __SCREAMING_SNAKE_CASE   				=							num_labels
      __SCREAMING_SNAKE_CASE   				=							scope





   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

      __SCREAMING_SNAKE_CASE   				=							None
      if self.use_labels:
         __SCREAMING_SNAKE_CASE   				=							ids_tensor([self.batch_size, self.image_size, self.image_size]					,self.num_labels     )

      __SCREAMING_SNAKE_CASE   				=							self.get_config()
      return config, pixel_values, labels





   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      return SegformerConfig(
          image_size=self.image_size					,num_channels=self.num_channels					,num_encoder_blocks=self.num_encoder_blocks					,depths=self.depths					,hidden_sizes=self.hidden_sizes					,num_attention_heads=self.num_attention_heads					,hidden_act=self.hidden_act					,hidden_dropout_prob=self.hidden_dropout_prob					,attention_probs_dropout_prob=self.attention_probs_dropout_prob					,initializer_range=self.initializer_range					,)





   def   UpperCAmelCase__  (       self					:      Any					,lowerCamelCase					:      int					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SegformerModel(config=lowerCamelCase     )
      model.to(lowerCamelCase     )
      model.eval()
      __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							__SCREAMING_SNAKE_CASE   				=							self.image_size // (self.downsampling_rates[-1] * 2)
      self.parent.assertEqual(
          result.last_hidden_state.shape					,(self.batch_size, self.hidden_sizes[-1], expected_height, expected_width)     )





   def   UpperCAmelCase__  (       self					:      Any					,lowerCamelCase					:      int					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.num_labels
      __SCREAMING_SNAKE_CASE   				=							SegformerForSemanticSegmentation(lowerCamelCase     )
      model.to(lowerCamelCase     )
      model.eval()
      __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase     )
      self.parent.assertEqual(
          result.logits.shape					,(self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4)     )
      __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase					,labels=lowerCamelCase     )
      self.parent.assertEqual(
          result.logits.shape					,(self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4)     )
      self.parent.assertGreater(result.loss					,0.0     )





   def   UpperCAmelCase__  (       self					:      str					,lowerCamelCase					:      Tuple					,lowerCamelCase					:      Optional[Any]					,lowerCamelCase					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							1
      __SCREAMING_SNAKE_CASE   				=							SegformerForSemanticSegmentation(config=lowerCamelCase     )
      model.to(lowerCamelCase     )
      model.eval()
      __SCREAMING_SNAKE_CASE   				=							torch.randint(0					,1					,(self.batch_size, self.image_size, self.image_size)     ).to(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase					,labels=lowerCamelCase     )
      self.parent.assertGreater(result.loss					,0.0     )





   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.prepare_config_and_inputs()
      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							config_and_inputs
      __SCREAMING_SNAKE_CASE   				=							{"""pixel_values""": pixel_values}
      return config, inputs_dict







@require_torch
class __a			( _snake_case, _snake_case, unittest.TestCase      ):
   __UpperCamelCase :			Dict						   =						(
       (
           SegformerModel,
           SegformerForSemanticSegmentation,
           SegformerForImageClassification,
       )
       if is_torch_available()
       else ()
   )
   __UpperCamelCase :			Union[str, Any]						   =						(
       {
           'feature-extraction': SegformerModel,
           'image-classification': SegformerForImageClassification,
           'image-segmentation': SegformerForSemanticSegmentation,
       }
       if is_torch_available()
       else {}
   )

   __UpperCamelCase :			Optional[Any]						   =						True
   __UpperCamelCase :			List[Any]						   =						False
   __UpperCamelCase :			Union[str, Any]						   =						False
   __UpperCamelCase :			Optional[int]						   =						False





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SegformerModelTester(self     )
      __SCREAMING_SNAKE_CASE   				=							SegformerConfigTester(self					,config_class=lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      self.config_tester.run_common_tests()





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_binary_image_segmentation(*lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_image_segmentation(*lowerCamelCase     )





   @unittest.skip("""SegFormer does not use inputs_embeds"""     )
   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      pass





   @unittest.skip("""SegFormer does not have get_input_embeddings method and get_output_embeddings methods"""     )
   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      pass





   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
         __SCREAMING_SNAKE_CASE   				=							model_class(lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							inspect.signature(model.forward     )
         # signature.parameters is an OrderedDict => so arg_names order is deterministic
         __SCREAMING_SNAKE_CASE   				=							[*signature.parameters.keys()]

         __SCREAMING_SNAKE_CASE   				=							["""pixel_values"""]
         self.assertListEqual(arg_names[:1]					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs_for_common()
      __SCREAMING_SNAKE_CASE   				=							True

      for model_class in self.all_model_classes:
         __SCREAMING_SNAKE_CASE   				=							True
         __SCREAMING_SNAKE_CASE   				=							False
         __SCREAMING_SNAKE_CASE   				=							True
         __SCREAMING_SNAKE_CASE   				=							model_class(lowerCamelCase     )
         model.to(lowerCamelCase     )
         model.eval()
         with torch.no_grad():
            __SCREAMING_SNAKE_CASE   				=							model(**self._prepare_for_class(lowerCamelCase					,lowerCamelCase     )     )
         __SCREAMING_SNAKE_CASE   				=							outputs.attentions

         __SCREAMING_SNAKE_CASE   				=							sum(self.model_tester.depths     )
         self.assertEqual(len(lowerCamelCase     )					,lowerCamelCase     )

         # check that output_attentions also work using config
         del inputs_dict["output_attentions"]
         __SCREAMING_SNAKE_CASE   				=							True
         __SCREAMING_SNAKE_CASE   				=							model_class(lowerCamelCase     )
         model.to(lowerCamelCase     )
         model.eval()
         with torch.no_grad():
            __SCREAMING_SNAKE_CASE   				=							model(**self._prepare_for_class(lowerCamelCase					,lowerCamelCase     )     )
         __SCREAMING_SNAKE_CASE   				=							outputs.attentions

         self.assertEqual(len(lowerCamelCase     )					,lowerCamelCase     )

         # verify the first attentions (first block, first layer)
         __SCREAMING_SNAKE_CASE   				=							(self.model_tester.image_size // 4) ** 2
         __SCREAMING_SNAKE_CASE   				=							(self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
         self.assertListEqual(
             list(attentions[0].shape[-3:]     )					,[self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]					,)

         # verify the last attentions (last block, last layer)
         __SCREAMING_SNAKE_CASE   				=							(self.model_tester.image_size // 32) ** 2
         __SCREAMING_SNAKE_CASE   				=							(self.model_tester.image_size // (32 * self.model_tester.sr_ratios[-1])) ** 2
         self.assertListEqual(
             list(attentions[-1].shape[-3:]     )					,[self.model_tester.num_attention_heads[-1], expected_seq_len, expected_reduced_seq_len]					,)
         __SCREAMING_SNAKE_CASE   				=							len(lowerCamelCase     )

         # Check attention is always last and order is fine
         __SCREAMING_SNAKE_CASE   				=							True
         __SCREAMING_SNAKE_CASE   				=							True
         __SCREAMING_SNAKE_CASE   				=							model_class(lowerCamelCase     )
         model.to(lowerCamelCase     )
         model.eval()
         with torch.no_grad():
            __SCREAMING_SNAKE_CASE   				=							model(**self._prepare_for_class(lowerCamelCase					,lowerCamelCase     )     )

         self.assertEqual(out_len + 1					,len(lowerCamelCase     )     )

         __SCREAMING_SNAKE_CASE   				=							outputs.attentions

         self.assertEqual(len(lowerCamelCase     )					,lowerCamelCase     )
         # verify the first attentions (first block, first layer)
         __SCREAMING_SNAKE_CASE   				=							(self.model_tester.image_size // 4) ** 2
         __SCREAMING_SNAKE_CASE   				=							(self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
         self.assertListEqual(
             list(self_attentions[0].shape[-3:]     )					,[self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]					,)





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      def check_hidden_states_output(lowerCamelCase					:      Tuple					,lowerCamelCase					:      List[Any]					,lowerCamelCase					:      Dict     ):
         __SCREAMING_SNAKE_CASE   				=							model_class(lowerCamelCase     )
         model.to(lowerCamelCase     )
         model.eval()

         with torch.no_grad():
            __SCREAMING_SNAKE_CASE   				=							model(**self._prepare_for_class(lowerCamelCase					,lowerCamelCase     )     )

         __SCREAMING_SNAKE_CASE   				=							outputs.hidden_states

         __SCREAMING_SNAKE_CASE   				=							self.model_tester.num_encoder_blocks
         self.assertEqual(len(lowerCamelCase     )					,lowerCamelCase     )

         # verify the first hidden states (first block)
         self.assertListEqual(
             list(hidden_states[0].shape[-3:]     )					,[
                 self.model_tester.hidden_sizes[0],
                 self.model_tester.image_size // 4,
                 self.model_tester.image_size // 4,
             ]					,)

      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
         __SCREAMING_SNAKE_CASE   				=							True
         check_hidden_states_output(lowerCamelCase					,lowerCamelCase					,lowerCamelCase     )

         # check that output_hidden_states also work using config
         del inputs_dict["output_hidden_states"]
         __SCREAMING_SNAKE_CASE   				=							True

         check_hidden_states_output(lowerCamelCase					,lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      if not self.model_tester.is_training:
         return

      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							self.model_tester.prepare_config_and_inputs_for_common()
      __SCREAMING_SNAKE_CASE   				=							True

      for model_class in self.all_model_classes:
         if model_class in get_values(lowerCamelCase     ):
            continue

         __SCREAMING_SNAKE_CASE   				=							model_class(lowerCamelCase     )
         model.to(lowerCamelCase     )
         model.train()
         __SCREAMING_SNAKE_CASE   				=							self._prepare_for_class(lowerCamelCase					,lowerCamelCase					,return_labels=lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							model(**lowerCamelCase     ).loss
         loss.backward()





   @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""     )
   def   UpperCAmelCase__  (       self					:      Union[str, Any]     ):

      '''simple docstring'''

      pass





   @slow
   def   UpperCAmelCase__  (       self					:      Union[str, Any]     ):

      '''simple docstring'''

      for model_name in SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         __SCREAMING_SNAKE_CASE   				=							SegformerModel.from_pretrained(lowerCamelCase     )
         self.assertIsNotNone(lowerCamelCase     )





def     __magic_name__							(				)    ->       List[str]:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE   				=							Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""   )
   return image







@require_torch
class __a			( unittest.TestCase      ):





   @slow
   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SegformerImageProcessor(
          image_scale=(512, 512)					,keep_ratio=lowerCamelCase					,align=lowerCamelCase					,do_random_crop=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							SegformerForSemanticSegmentation.from_pretrained("""nvidia/segformer-b0-finetuned-ade-512-512"""     ).to(
          lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							prepare_img()
      __SCREAMING_SNAKE_CASE   				=							image_processor(images=lowerCamelCase					,return_tensors="""pt"""     )
      __SCREAMING_SNAKE_CASE   				=							encoded_inputs.pixel_values.to(lowerCamelCase     )

      with torch.no_grad():
         __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							torch.Size((1, model.config.num_labels, 128, 128)     )
      self.assertEqual(outputs.logits.shape					,lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							torch.tensor(
          [
              [[-4.6_310, -5.5_232, -6.2_356], [-5.1_921, -6.1_444, -6.5_996], [-5.4_424, -6.2_790, -6.7_574]],
              [[-12.1_391, -13.3_122, -13.9_554], [-12.8_732, -13.9_352, -14.3_563], [-12.9_438, -13.8_226, -14.2_513]],
              [[-12.5_134, -13.4_686, -14.4_915], [-12.8_669, -14.4_343, -14.7_758], [-13.2_523, -14.5_819, -15.0_694]],
          ]     ).to(lowerCamelCase     )
      self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]					,lowerCamelCase					,atol=1E-4     )     )





   @slow
   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SegformerImageProcessor(
          image_scale=(512, 512)					,keep_ratio=lowerCamelCase					,align=lowerCamelCase					,do_random_crop=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							SegformerForSemanticSegmentation.from_pretrained(
          """nvidia/segformer-b1-finetuned-cityscapes-1024-1024"""     ).to(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							prepare_img()
      __SCREAMING_SNAKE_CASE   				=							image_processor(images=lowerCamelCase					,return_tensors="""pt"""     )
      __SCREAMING_SNAKE_CASE   				=							encoded_inputs.pixel_values.to(lowerCamelCase     )

      with torch.no_grad():
         __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							torch.Size((1, model.config.num_labels, 128, 128)     )
      self.assertEqual(outputs.logits.shape					,lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							torch.tensor(
          [
              [[-13.5_748, -13.9_111, -12.6_500], [-14.3_500, -15.3_683, -14.2_328], [-14.7_532, -16.0_424, -15.6_087]],
              [[-17.1_651, -15.8_725, -12.9_653], [-17.2_580, -17.3_718, -14.8_223], [-16.6_058, -16.8_783, -16.7_452]],
              [[-3.6_456, -3.0_209, -1.4_203], [-3.0_797, -3.1_959, -2.0_000], [-1.8_757, -1.9_217, -1.6_997]],
          ]     ).to(lowerCamelCase     )
      self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]					,lowerCamelCase					,atol=1E-1     )     )





   @slow
   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SegformerImageProcessor(
          image_scale=(512, 512)					,keep_ratio=lowerCamelCase					,align=lowerCamelCase					,do_random_crop=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							SegformerForSemanticSegmentation.from_pretrained("""nvidia/segformer-b0-finetuned-ade-512-512"""     ).to(
          lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							prepare_img()
      __SCREAMING_SNAKE_CASE   				=							image_processor(images=lowerCamelCase					,return_tensors="""pt"""     )
      __SCREAMING_SNAKE_CASE   				=							encoded_inputs.pixel_values.to(lowerCamelCase     )

      with torch.no_grad():
         __SCREAMING_SNAKE_CASE   				=							model(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							outputs.logits.detach().cpu()

      __SCREAMING_SNAKE_CASE   				=							image_processor.post_process_semantic_segmentation(outputs=lowerCamelCase					,target_sizes=[(500, 300)]     )
      __SCREAMING_SNAKE_CASE   				=							torch.Size((500, 300)     )
      self.assertEqual(segmentation[0].shape					,lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							image_processor.post_process_semantic_segmentation(outputs=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							torch.Size((128, 128)     )
      self.assertEqual(segmentation[0].shape					,lowerCamelCase     )












'''simple docstring'''

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


a   	=    logging.get_logger(__name__)

a   	=    {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

a   	=    {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

a   	=    {
    "abeja/gpt-neox-japanese-2.7b": 2048,
}

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase   )    ->       Any:

   '''simple docstring'''







   with open(__UpperCAmelCase				,						"""r"""				,						encoding="""utf-8"""   ) as f:
      __SCREAMING_SNAKE_CASE   				=							json.loads(f.read()   )

   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   with open(__UpperCAmelCase				,						"""r"""				,						encoding="""utf-8"""   ) as f:
      __SCREAMING_SNAKE_CASE   				=							f.readlines()
   __SCREAMING_SNAKE_CASE   				=							[[t.rstrip("""\n"""   )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""   ).split(""","""   ) for t in token]
   for idx, b in enumerate(__UpperCAmelCase   ):
      __SCREAMING_SNAKE_CASE   				=							b
      __SCREAMING_SNAKE_CASE   				=							idx
      for wd in b:
         __SCREAMING_SNAKE_CASE   				=							idx

   return vocab, raw_vocab, ids_to_tokens, emoji







class __a			( _snake_case      ):
   __UpperCamelCase :			Tuple						   =						VOCAB_FILES_NAMES
   __UpperCamelCase :			Tuple						   =						PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase :			Any						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase :			Union[str, Any]						   =						['input_ids', 'attention_mask']





   def __init__(       self					:      Dict					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      int					,lowerCamelCase					:      str="<|endoftext|>"					,lowerCamelCase					:      List[Any]="<|endoftext|>"					,lowerCamelCase					:      str="<|startoftext|>"					,lowerCamelCase					:      Dict="<|endoftext|>"					,lowerCamelCase					:      int=False					,**lowerCamelCase					:      str					,):

      '''simple docstring'''

      super().__init__(
          unk_token=lowerCamelCase					,pad_token=lowerCamelCase					,bos_token=lowerCamelCase					,eos_token=lowerCamelCase					,do_clean_text=lowerCamelCase					,**lowerCamelCase					,)
      if not os.path.isfile(lowerCamelCase     ):
         raise ValueError(
             f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
             """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""     )
      if not os.path.isfile(lowerCamelCase     ):
         raise ValueError(
             f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
             """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""     )
      __SCREAMING_SNAKE_CASE   				=							do_clean_text
      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							load_vocab_and_emoji(lowerCamelCase					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							SubWordJapaneseTokenizer(
          vocab=self.vocab					,ids_to_tokens=self.ids_to_tokens					,emoji=self.emoji     )





   @property
   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      return len(self.raw_vocab     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      return dict(self.raw_vocab					,**self.added_tokens_encoder     )





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      return self.subword_tokenizer.tokenize(lowerCamelCase					,clean=self.do_clean_text     )





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      Union[str, Any]     ):

      '''simple docstring'''

      return self.vocab.get(lowerCamelCase					,self.vocab.get(self.unk_token     )     )





   def   UpperCAmelCase__  (       self					:      List[Any]					,lowerCamelCase					:      Optional[Any]     ):

      '''simple docstring'''

      return self.subword_tokenizer.convert_id_to_token(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""""".join(lowerCamelCase     ).strip()
      return out_string





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      "Conversation"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(lowerCamelCase					,add_special_tokens=lowerCamelCase     ) + [self.eos_token_id]     )

      if len(lowerCamelCase     ) > self.model_max_length:
         __SCREAMING_SNAKE_CASE   				=							input_ids[-self.model_max_length :]
      return input_ids





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      str					,lowerCamelCase					:      Optional[str] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							0
      if os.path.isdir(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							os.path.join(
             lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )
         __SCREAMING_SNAKE_CASE   				=							os.path.join(
             lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]     )
      else:
         __SCREAMING_SNAKE_CASE   				=							(
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
         )
         __SCREAMING_SNAKE_CASE   				=							(
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
         )
      with open(lowerCamelCase					,"""w"""					,encoding="""utf-8"""     ) as writer:
         for token_index, token in self.ids_to_tokens.items():
            if index != token_index:
               logger.warning(
                   f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                   """ Please check that the vocabulary is not corrupted!"""     )
               __SCREAMING_SNAKE_CASE   				=							token_index
            writer.write(""",""".join(lowerCamelCase     ) + """\n"""     )
            index += 1
      with open(lowerCamelCase					,"""w"""					,encoding="""utf-8"""     ) as writer:
         json.dump(self.emoji					,lowerCamelCase     )
      return vocab_file, emoji_file







class __a			( _snake_case      ):





   def __init__(       self					:      Tuple					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							vocab  # same as swe
      __SCREAMING_SNAKE_CASE   				=							ids_to_tokens  # same as bpe
      __SCREAMING_SNAKE_CASE   				=							emoji
      __SCREAMING_SNAKE_CASE   				=							np.max([len(lowerCamelCase     ) for w in self.vocab.keys()]     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""     )
      __SCREAMING_SNAKE_CASE   				=							"""─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
      __SCREAMING_SNAKE_CASE   				=							"""▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
      __SCREAMING_SNAKE_CASE   				=							str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}     )





   def __len__(       self					:      List[str]     ):

      '''simple docstring'''

      return len(self.ids_to_tokens     )





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<URL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<EMAIL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<TEL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<DATE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<DATE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<PRICE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							content.translate(self.content_transa     )
      while "<BLOCK><BLOCK>" in content:
         __SCREAMING_SNAKE_CASE   				=							content.replace("""<BLOCK><BLOCK>"""					,"""<BLOCK>"""     )
      return content





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      Optional[Any]					,lowerCamelCase					:      List[str]=False     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							text.replace(""" """					,"""<SP>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""　"""					,"""<SP>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\r\n"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\n"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\r"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\t"""					,"""<TAB>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""—"""					,"""ー"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""−"""					,"""ー"""     )
      for k, v in self.emoji["emoji"].items():
         if k in text:
            __SCREAMING_SNAKE_CASE   				=							text.replace(lowerCamelCase					,lowerCamelCase     )
      if clean:
         __SCREAMING_SNAKE_CASE   				=							self.clean_text(lowerCamelCase     )

      def check_simbol(lowerCamelCase					:      List[str]     ):
         __SCREAMING_SNAKE_CASE   				=							x.encode()
         if len(lowerCamelCase     ) == 1 and len(lowerCamelCase     ) == 2:
            __SCREAMING_SNAKE_CASE   				=							(int(e[0]     ) << 8) + int(e[1]     )
            if (
                (c >= 0xC_2A1 and c <= 0xC_2BF)
                or (c >= 0xC_780 and c <= 0xC_783)
                or (c >= 0xC_AB9 and c <= 0xC_BBF)
                or (c >= 0xC_C80 and c <= 0xC_DA2)
            ):
               return True
         return False

      def checkuae(lowerCamelCase					:      Union[str, Any]     ):
         __SCREAMING_SNAKE_CASE   				=							x.encode()
         if len(lowerCamelCase     ) == 1 and len(lowerCamelCase     ) == 3:
            __SCREAMING_SNAKE_CASE   				=							(int(e[0]     ) << 16) + (int(e[1]     ) << 8) + int(e[2]     )
            if c >= 0xE28_080 and c <= 0xE2B_07F:
               return True
         return False

      __SCREAMING_SNAKE_CASE   				=							0
      __SCREAMING_SNAKE_CASE   				=							[]
      while pos < len(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							min(len(lowerCamelCase     )					,pos + self.maxlen + 1     ) if text[pos] == """<""" else pos + 3
         __SCREAMING_SNAKE_CASE   				=							[]  # (token_id, token, pos)
         for e in range(lowerCamelCase					,lowerCamelCase					,-1     ):
            __SCREAMING_SNAKE_CASE   				=							text[pos:e]
            if wd in self.vocab:
               if wd[0] == "<" and len(lowerCamelCase     ) > 2:
                  __SCREAMING_SNAKE_CASE   				=							[(self.vocab[wd], wd, e)]
                  break
               else:
                  candidates.append((self.vocab[wd], wd, e)     )
         if len(lowerCamelCase     ) > 0:
            # the smallest token_id is adopted
            __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							sorted(lowerCamelCase					,key=lambda lowerCamelCase     : x[0]     )[0]
            result.append(lowerCamelCase     )
            __SCREAMING_SNAKE_CASE   				=							e
         else:
            __SCREAMING_SNAKE_CASE   				=							pos + 1
            __SCREAMING_SNAKE_CASE   				=							text[pos:end]
            if check_simbol(lowerCamelCase     ):
               result.append("""<KIGOU>"""     )
            elif checkuae(lowerCamelCase     ):
               result.append("""<U2000U2BFF>"""     )
            else:
               for i in wd.encode("""utf-8"""     ):
                  result.append("""<|byte%d|>""" % i     )
            __SCREAMING_SNAKE_CASE   				=							end
      return result





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      Dict					,lowerCamelCase					:      Optional[Any]="\n"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
         byte_tokens.append(int(word[6:-2]     )     )
      else:
         if len(lowerCamelCase     ) > 0:
            words.append(bytearray(lowerCamelCase     ).decode("""utf-8"""					,errors="""replace"""     )     )
            __SCREAMING_SNAKE_CASE   				=							[]
         if word[:7] == "<|emoji" and word[-2:] == "|>":
            words.append(self.emoji["""emoji_inv"""][word]     )
         elif word == "<SP>":
            words.append(""" """     )
         elif word == "<BR>":
            words.append(lowerCamelCase     )
         elif word == "<TAB>":
            words.append("""\t"""     )
         elif word == "<BLOCK>":
            words.append("""▀"""     )
         elif word == "<KIGOU>":
            words.append("""ǀ"""     )
         elif word == "<U2000U2BFF>":
            words.append("""‖"""     )
         else:
            words.append(lowerCamelCase     )
      if len(lowerCamelCase     ) > 0:
         words.append(bytearray(lowerCamelCase     ).decode("""utf-8"""					,errors="""replace"""     )     )
      __SCREAMING_SNAKE_CASE   				=							"""""".join(lowerCamelCase     )
      return text





