


'''simple docstring'''



import json
import pathlib
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import DetaImageProcessor





class   UpperCAmelCase_    (unittest.TestCase    ):





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=7     ,	SCREAMING_SNAKE_CASE_=3     ,	SCREAMING_SNAKE_CASE_=30     ,	SCREAMING_SNAKE_CASE_=4_00     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=[0.5, 0.5, 0.5]     ,	SCREAMING_SNAKE_CASE_=[0.5, 0.5, 0.5]     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=1 / 2_55     ,	SCREAMING_SNAKE_CASE_=True     ,	)							->	int:
         # by setting size["longest_edge"] > max_resolution we're effectively not testing this :p
         __lowerCamelCase  :      int														=						size if size is not None else {'shortest_edge': 18, 'longest_edge': 13_33}
         __lowerCamelCase  :      Optional[int]														=						parent
         __lowerCamelCase  :      Union[str, Any]														=						batch_size
         __lowerCamelCase  :      str														=						num_channels
         __lowerCamelCase  :      Union[str, Any]														=						min_resolution
         __lowerCamelCase  :      int														=						max_resolution
         __lowerCamelCase  :      str														=						do_resize
         __lowerCamelCase  :      Tuple														=						size
         __lowerCamelCase  :      str														=						do_normalize
         __lowerCamelCase  :      Optional[int]														=						image_mean
         __lowerCamelCase  :      int														=						image_std
         __lowerCamelCase  :      Union[str, Any]														=						do_rescale
         __lowerCamelCase  :      Optional[Any]														=						rescale_factor
         __lowerCamelCase  :      Optional[int]														=						do_pad







    def  lowercase_							(				self      )							->	str:
         return {
             "do_resize": self.do_resize,
             "size": self.size,
             "do_normalize": self.do_normalize,
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_rescale": self.do_rescale,
             "rescale_factor": self.rescale_factor,
             "do_pad": self.do_pad,
         }







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=False      )							->	Any:
         if not batched:
              __lowerCamelCase  :      str														=						image_inputs[0]
              if isinstance(SCREAMING_SNAKE_CASE_     ,	Image.Image      ):
                   __lowerCamelCase							,			__lowerCamelCase  :      Union[str, Any]														=						image.size
              else:
                   __lowerCamelCase							,			__lowerCamelCase  :      Optional[int]														=						image.shape[1], image.shape[2]
              if w < h:
                   __lowerCamelCase  :      List[Any]														=						int(self.size['shortest_edge'] * h / w      )
                   __lowerCamelCase  :      str														=						self.size['shortest_edge']
              elif w > h:
                   __lowerCamelCase  :      Optional[int]														=						self.size['shortest_edge']
                   __lowerCamelCase  :      List[str]														=						int(self.size['shortest_edge'] * w / h      )
              else:
                   __lowerCamelCase  :      Any														=						self.size['shortest_edge']
                   __lowerCamelCase  :      List[str]														=						self.size['shortest_edge']

         else:
              __lowerCamelCase  :      Optional[Any]														=						[]
              for image in image_inputs:
                   __lowerCamelCase							,			__lowerCamelCase  :      Dict														=						self.get_expected_values([image]      )
                   expected_values.append((expected_height, expected_width)      )
              __lowerCamelCase  :      Any														=						max(SCREAMING_SNAKE_CASE_     ,	key=lambda SCREAMING_SNAKE_CASE_      : item[0]      )[0]
              __lowerCamelCase  :      Any														=						max(SCREAMING_SNAKE_CASE_     ,	key=lambda SCREAMING_SNAKE_CASE_      : item[1]      )[1]

         return expected_height, expected_width







@require_torch
@require_vision
class   UpperCAmelCase_    (_UpperCAmelCase   ,						unittest.TestCase    ):





    """simple docstring"""






    lowerCamelCase    :   Union[str, Any]					=     DetaImageProcessor if is_vision_available() else None







    def  lowercase_							(				self      )							->	str:
         __lowerCamelCase  :      Dict														=						DetaImageProcessingTester(self      )







    @property
    def  lowercase_							(				self      )							->	Tuple:
         return self.image_processor_tester.prepare_image_processor_dict()







    def  lowercase_							(				self      )							->	str:
         __lowerCamelCase  :      Dict														=						self.image_processing_class(**self.image_processor_dict      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'image_mean'      )      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'image_std'      )      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'do_normalize'      )      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'do_resize'      )      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'do_rescale'      )      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'do_pad'      )      )
         self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	'size'      )      )







    def  lowercase_							(				self      )							->	int:
         __lowerCamelCase  :      Dict														=						self.image_processing_class.from_dict(self.image_processor_dict      )
         self.assertEqual(image_processor.size     ,	{'shortest_edge': 18, 'longest_edge': 13_33}      )
         self.assertEqual(image_processor.do_pad     ,	SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	List[str]:
         pass







    def  lowercase_							(				self      )							->	Union[str, Any]:
         # Initialize image_processing
         __lowerCamelCase  :      List[Any]														=						self.image_processing_class(**self.image_processor_dict      )
         # create random PIL images
         __lowerCamelCase  :      Dict														=						prepare_image_inputs(self.image_processor_tester     ,	equal_resolution=SCREAMING_SNAKE_CASE_      )
         for image in image_inputs:
              self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	Image.Image      )

         # Test not batched input
         __lowerCamelCase  :      Dict														=						image_processing(image_inputs[0]     ,	return_tensors='pt'      ).pixel_values

         __lowerCamelCase							,			__lowerCamelCase  :      List[str]														=						self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_      )

         self.assertEqual(
             encoded_images.shape     ,	(1, self.image_processor_tester.num_channels, expected_height, expected_width)     ,	)

         # Test batched
         __lowerCamelCase							,			__lowerCamelCase  :      Optional[int]														=						self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_     ,	batched=SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      str														=						image_processing(SCREAMING_SNAKE_CASE_     ,	return_tensors='pt'      ).pixel_values
         self.assertEqual(
             encoded_images.shape     ,	(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )     ,	)







    def  lowercase_							(				self      )							->	Tuple:
         # Initialize image_processing
         __lowerCamelCase  :      List[Any]														=						self.image_processing_class(**self.image_processor_dict      )
         # create random numpy tensors
         __lowerCamelCase  :      Union[str, Any]														=						prepare_image_inputs(self.image_processor_tester     ,	equal_resolution=SCREAMING_SNAKE_CASE_     ,	numpify=SCREAMING_SNAKE_CASE_      )
         for image in image_inputs:
              self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	np.ndarray      )

         # Test not batched input
         __lowerCamelCase  :      List[Any]														=						image_processing(image_inputs[0]     ,	return_tensors='pt'      ).pixel_values

         __lowerCamelCase							,			__lowerCamelCase  :      int														=						self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_      )

         self.assertEqual(
             encoded_images.shape     ,	(1, self.image_processor_tester.num_channels, expected_height, expected_width)     ,	)

         # Test batched
         __lowerCamelCase  :      Tuple														=						image_processing(SCREAMING_SNAKE_CASE_     ,	return_tensors='pt'      ).pixel_values

         __lowerCamelCase							,			__lowerCamelCase  :      List[str]														=						self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_     ,	batched=SCREAMING_SNAKE_CASE_      )

         self.assertEqual(
             encoded_images.shape     ,	(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )     ,	)







    def  lowercase_							(				self      )							->	Optional[Any]:
         # Initialize image_processing
         __lowerCamelCase  :      List[Any]														=						self.image_processing_class(**self.image_processor_dict      )
         # create random PyTorch tensors
         __lowerCamelCase  :      Optional[Any]														=						prepare_image_inputs(self.image_processor_tester     ,	equal_resolution=SCREAMING_SNAKE_CASE_     ,	torchify=SCREAMING_SNAKE_CASE_      )
         for image in image_inputs:
              self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	torch.Tensor      )

         # Test not batched input
         __lowerCamelCase  :      Optional[Any]														=						image_processing(image_inputs[0]     ,	return_tensors='pt'      ).pixel_values

         __lowerCamelCase							,			__lowerCamelCase  :      Optional[Any]														=						self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_      )

         self.assertEqual(
             encoded_images.shape     ,	(1, self.image_processor_tester.num_channels, expected_height, expected_width)     ,	)

         # Test batched
         __lowerCamelCase  :      Optional[Any]														=						image_processing(SCREAMING_SNAKE_CASE_     ,	return_tensors='pt'      ).pixel_values

         __lowerCamelCase							,			__lowerCamelCase  :      str														=						self.image_processor_tester.get_expected_values(SCREAMING_SNAKE_CASE_     ,	batched=SCREAMING_SNAKE_CASE_      )

         self.assertEqual(
             encoded_images.shape     ,	(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )     ,	)







    @slow
    def  lowercase_							(				self      )							->	List[Any]:
         # prepare image and target
         __lowerCamelCase  :      Dict														=						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'      )
         with open('./tests/fixtures/tests_samples/COCO/coco_annotations.txt'     ,	'r'      ) as f:
              __lowerCamelCase  :      List[Any]														=						json.loads(f.read()      )

         __lowerCamelCase  :      List[str]														=						{'image_id': 3_97_69, 'annotations': target}

         # encode them
         __lowerCamelCase  :      Union[str, Any]														=						DetaImageProcessor()
         __lowerCamelCase  :      int														=						image_processing(images=SCREAMING_SNAKE_CASE_     ,	annotations=SCREAMING_SNAKE_CASE_     ,	return_tensors='pt'      )

         # verify pixel values
         __lowerCamelCase  :      Optional[int]														=						torch.Size([1, 3, 8_00, 10_66]      )
         self.assertEqual(encoding['pixel_values'].shape     ,	SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      int														=						torch.tensor([0.2_7_9_6, 0.3_1_3_8, 0.3_4_8_1]      )
         self.assertTrue(torch.allclose(encoding['pixel_values'][0, 0, 0, :3]     ,	SCREAMING_SNAKE_CASE_     ,	atol=1E-4      )      )

         # verify area
         __lowerCamelCase  :      Optional[Any]														=						torch.tensor([5_8_8_7.9_6_0_0, 1_1_2_5_0.2_0_6_1, 4_8_9_3_5_3.8_4_3_8, 8_3_7_1_2_2.7_5_0_0, 1_4_7_9_6_7.5_1_5_6, 1_6_5_7_3_2.3_4_3_8]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['area']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify boxes
         __lowerCamelCase  :      Optional[Any]														=						torch.Size([6, 4]      )
         self.assertEqual(encoding['labels'][0]['boxes'].shape     ,	SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      int														=						torch.tensor([0.5_5_0_3, 0.2_7_6_5, 0.0_6_0_4, 0.2_2_1_5]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['boxes'][0]     ,	SCREAMING_SNAKE_CASE_     ,	atol=1E-3      )      )
         # verify image_id
         __lowerCamelCase  :      Union[str, Any]														=						torch.tensor([3_97_69]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['image_id']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify is_crowd
         __lowerCamelCase  :      List[Any]														=						torch.tensor([0, 0, 0, 0, 0, 0]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['iscrowd']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify class_labels
         __lowerCamelCase  :      Optional[int]														=						torch.tensor([75, 75, 63, 65, 17, 17]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['class_labels']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify orig_size
         __lowerCamelCase  :      List[Any]														=						torch.tensor([4_80, 6_40]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['orig_size']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify size
         __lowerCamelCase  :      Optional[Any]														=						torch.tensor([8_00, 10_66]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['size']     ,	SCREAMING_SNAKE_CASE_      )      )







    @slow
    def  lowercase_							(				self      )							->	Optional[Any]:
         # prepare image, target and masks_path
         __lowerCamelCase  :      List[str]														=						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'      )
         with open('./tests/fixtures/tests_samples/COCO/coco_panoptic_annotations.txt'     ,	'r'      ) as f:
              __lowerCamelCase  :      Any														=						json.loads(f.read()      )

         __lowerCamelCase  :      int														=						{'file_name': '000000039769.png', 'image_id': 3_97_69, 'segments_info': target}

         __lowerCamelCase  :      List[Any]														=						pathlib.Path('./tests/fixtures/tests_samples/COCO/coco_panoptic'      )

         # encode them
         __lowerCamelCase  :      Union[str, Any]														=						DetaImageProcessor(format='coco_panoptic'      )
         __lowerCamelCase  :      Optional[Any]														=						image_processing(images=SCREAMING_SNAKE_CASE_     ,	annotations=SCREAMING_SNAKE_CASE_     ,	masks_path=SCREAMING_SNAKE_CASE_     ,	return_tensors='pt'      )

         # verify pixel values
         __lowerCamelCase  :      List[str]														=						torch.Size([1, 3, 8_00, 10_66]      )
         self.assertEqual(encoding['pixel_values'].shape     ,	SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      int														=						torch.tensor([0.2_7_9_6, 0.3_1_3_8, 0.3_4_8_1]      )
         self.assertTrue(torch.allclose(encoding['pixel_values'][0, 0, 0, :3]     ,	SCREAMING_SNAKE_CASE_     ,	atol=1E-4      )      )

         # verify area
         __lowerCamelCase  :      Optional[Any]														=						torch.tensor([1_4_7_9_7_9.6_8_7_5, 1_6_5_5_2_7.0_4_6_9, 4_8_4_6_3_8.5_9_3_8, 1_1_2_9_2.9_3_7_5, 5_8_7_9.6_5_6_2, 7_6_3_4.1_1_4_7]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['area']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify boxes
         __lowerCamelCase  :      int														=						torch.Size([6, 4]      )
         self.assertEqual(encoding['labels'][0]['boxes'].shape     ,	SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Optional[Any]														=						torch.tensor([0.2_6_2_5, 0.5_4_3_7, 0.4_6_8_8, 0.8_6_2_5]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['boxes'][0]     ,	SCREAMING_SNAKE_CASE_     ,	atol=1E-3      )      )
         # verify image_id
         __lowerCamelCase  :      Optional[int]														=						torch.tensor([3_97_69]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['image_id']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify is_crowd
         __lowerCamelCase  :      Union[str, Any]														=						torch.tensor([0, 0, 0, 0, 0, 0]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['iscrowd']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify class_labels
         __lowerCamelCase  :      Optional[Any]														=						torch.tensor([17, 17, 63, 75, 75, 93]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['class_labels']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify masks
         __lowerCamelCase  :      Any														=						82_28_73
         self.assertEqual(encoding['labels'][0]['masks'].sum().item()     ,	SCREAMING_SNAKE_CASE_      )
         # verify orig_size
         __lowerCamelCase  :      Tuple														=						torch.tensor([4_80, 6_40]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['orig_size']     ,	SCREAMING_SNAKE_CASE_      )      )
         # verify size
         __lowerCamelCase  :      Tuple														=						torch.tensor([8_00, 10_66]      )
         self.assertTrue(torch.allclose(encoding['labels'][0]['size']     ,	SCREAMING_SNAKE_CASE_      )      )










'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A__						:   str				=    logging.get_logger(__name__)

A__						:   str				=    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

A__						:   Tuple				=    {
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

A__						:   str				=    {
    """junnyu/roformer_chinese_small""": 1536,
    """junnyu/roformer_chinese_base""": 1536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


A__						:   Tuple				=    {
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    lowerCamelCase    :   Union[str, Any]					=     VOCAB_FILES_NAMES
    lowerCamelCase    :   int					=     PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase    :   Any					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase    :   Optional[int]					=     PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase    :   Dict					=     RoFormerTokenizer







    def __init__(				self     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_="[UNK]"     ,	SCREAMING_SNAKE_CASE_="[SEP]"     ,	SCREAMING_SNAKE_CASE_="[PAD]"     ,	SCREAMING_SNAKE_CASE_="[CLS]"     ,	SCREAMING_SNAKE_CASE_="[MASK]"     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Optional[Any]:
         super().__init__(
             SCREAMING_SNAKE_CASE_     ,	tokenizer_file=SCREAMING_SNAKE_CASE_     ,	do_lower_case=SCREAMING_SNAKE_CASE_     ,	unk_token=SCREAMING_SNAKE_CASE_     ,	sep_token=SCREAMING_SNAKE_CASE_     ,	pad_token=SCREAMING_SNAKE_CASE_     ,	cls_token=SCREAMING_SNAKE_CASE_     ,	mask_token=SCREAMING_SNAKE_CASE_     ,	tokenize_chinese_chars=SCREAMING_SNAKE_CASE_     ,	strip_accents=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)

         __lowerCamelCase  :      Optional[int]														=						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
         if (
             pre_tok_state.get('lowercase'     ,	SCREAMING_SNAKE_CASE_      ) != do_lower_case
             or pre_tok_state.get('strip_accents'     ,	SCREAMING_SNAKE_CASE_      ) != strip_accents
         ):
              __lowerCamelCase  :      Optional[int]														=						getattr(SCREAMING_SNAKE_CASE_     ,	pre_tok_state.pop('type'      )      )
              __lowerCamelCase  :      Union[str, Any]														=						do_lower_case
              __lowerCamelCase  :      str														=						strip_accents
              __lowerCamelCase  :      Optional[Any]														=						pre_tok_class(**SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      Tuple														=						do_lower_case







    def __getstate__(				self      )							->	List[str]:
         __lowerCamelCase  :      Union[str, Any]														=						self.__dict__.copy()
         __lowerCamelCase  :      Dict														=						BertPreTokenizer()
         return state







    def __setstate__(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Optional[Any]:
         __lowerCamelCase  :      Optional[int]														=						d
         __lowerCamelCase  :      List[Any]														=						self.__dict__['_tokenizer'].get_vocab()
         __lowerCamelCase  :      Union[str, Any]														=						PreTokenizer.custom(JiebaPreTokenizer(SCREAMING_SNAKE_CASE_      )      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=None      )							->	str:
         __lowerCamelCase  :      Union[str, Any]														=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

         return output







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	List[int]:
         __lowerCamelCase  :      List[str]														=						[self.sep_token_id]
         __lowerCamelCase  :      Dict														=						[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	Tuple[str]:
         __lowerCamelCase  :      Optional[Any]														=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE_     ,	name=SCREAMING_SNAKE_CASE_      )
         return tuple(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=False     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Any:
         __lowerCamelCase  :      Tuple														=						BertPreTokenizer()
         return super().save_pretrained(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )






