"""simple docstring"""





import json
import pathlib
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import DetrImageProcessor
class  _snake_case     (					unittest.TestCase		):






						def __init__(       self      :			Optional[int]							,					UpperCAmelCase      :			Any							,					UpperCAmelCase      :			Any=7							,					UpperCAmelCase      :			int=3							,					UpperCAmelCase      :			Union[str, Any]=30							,					UpperCAmelCase      :			Tuple=400							,					UpperCAmelCase      :			List[Any]=True							,					UpperCAmelCase      :			Optional[Any]=None							,					UpperCAmelCase      :			int=True							,					UpperCAmelCase      :			Union[str, Any]=1 / 255							,					UpperCAmelCase      :			int=True							,					UpperCAmelCase      :			Dict=[0.5, 0.5, 0.5]							,					UpperCAmelCase      :			Optional[int]=[0.5, 0.5, 0.5]							,					UpperCAmelCase      :			List[str]=True							,					):
													# by setting size["longest_edge"] > max_resolution we're effectively not testing this :p
													__lowerCamelCase					:		Optional[Any]		   =     size if size is not None else {"shortest_edge": 18, "longest_edge": 1333}
													__lowerCamelCase					:		List[Any]		   =     parent
													__lowerCamelCase					:		int		   =     batch_size
													__lowerCamelCase					:		List[Any]		   =     num_channels
													__lowerCamelCase					:		Tuple		   =     min_resolution
													__lowerCamelCase					:		Tuple		   =     max_resolution
													__lowerCamelCase					:		List[Any]		   =     do_resize
													__lowerCamelCase					:		str		   =     size
													__lowerCamelCase					:		str		   =     do_rescale
													__lowerCamelCase					:		Tuple		   =     rescale_factor
													__lowerCamelCase					:		List[Any]		   =     do_normalize
													__lowerCamelCase					:		str		   =     image_mean
													__lowerCamelCase					:		Dict		   =     image_std
													__lowerCamelCase					:		Union[str, Any]		   =     do_pad






						def       lowerCamelCase__     (       self      :			Any    ):
													return {
													    "do_resize": self.do_resize,
													    "size": self.size,
													    "do_rescale": self.do_rescale,
													    "rescale_factor": self.rescale_factor,
													    "do_normalize": self.do_normalize,
													    "image_mean": self.image_mean,
													    "image_std": self.image_std,
													    "do_pad": self.do_pad,
													}







						def       lowerCamelCase__     (       self      :			List[str]							,					UpperCAmelCase      :			Union[str, Any]							,					UpperCAmelCase      :			str=False    ):
													if not batched:
																				__lowerCamelCase					:		int		   =     image_inputs[0]
																				if isinstance(UpperCAmelCase							,					Image.Image    ):
																											__lowerCamelCase   ,  __lowerCamelCase					:		int		   =     image.size
																				else:
																											__lowerCamelCase   ,  __lowerCamelCase					:		Optional[int]		   =     image.shape[1], image.shape[2]
																				if w < h:
																											__lowerCamelCase					:		Optional[Any]		   =     int(self.size["shortest_edge"] * h / w    )
																											__lowerCamelCase					:		Union[str, Any]		   =     self.size["shortest_edge"]
																				elif w > h:
																											__lowerCamelCase					:		int		   =     self.size["shortest_edge"]
																											__lowerCamelCase					:		int		   =     int(self.size["shortest_edge"] * w / h    )
																				else:
																											__lowerCamelCase					:		List[Any]		   =     self.size["shortest_edge"]
																											__lowerCamelCase					:		str		   =     self.size["shortest_edge"]

													else:
																				__lowerCamelCase					:		Any		   =     []
																				for image in image_inputs:
																											__lowerCamelCase   ,  __lowerCamelCase					:		Optional[int]		   =     self.get_expected_values([image]    )
																											expected_values.append((expected_height, expected_width)    )
																				__lowerCamelCase					:		Optional[int]		   =     max(UpperCAmelCase							,					key=lambda UpperCAmelCase    : item[0]    )[0]
																				__lowerCamelCase					:		Optional[int]		   =     max(UpperCAmelCase							,					key=lambda UpperCAmelCase    : item[1]    )[1]

													return expected_height, expected_width

@require_torch
@require_vision
class  _snake_case     (					a__				, unittest.TestCase		):
						snake_case__			  =			DetrImageProcessor if is_vision_available() else None






						def       lowerCamelCase__     (       self      :			List[Any]    ):
													__lowerCamelCase					:		str		   =     DetrImageProcessingTester(self    )






						@property
						def       lowerCamelCase__     (       self      :			Any    ):
													return self.image_processor_tester.prepare_image_processor_dict()






						def       lowerCamelCase__     (       self      :			List[Any]    ):
													__lowerCamelCase					:		Union[str, Any]		   =     self.image_processing_class(**self.image_processor_dict    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"image_mean"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"image_std"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"do_normalize"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"do_rescale"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"rescale_factor"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"do_resize"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"size"    )    )
													self.assertTrue(hasattr(UpperCAmelCase							,					"do_pad"    )    )






						def       lowerCamelCase__     (       self      :			Dict    ):
													__lowerCamelCase					:		Any		   =     self.image_processing_class.from_dict(self.image_processor_dict    )
													self.assertEqual(image_processor.size							,					{"shortest_edge": 18, "longest_edge": 1333}    )
													self.assertEqual(image_processor.do_pad							,					UpperCAmelCase    )

													__lowerCamelCase					:		Union[str, Any]		   =     self.image_processing_class.from_dict(
													    self.image_processor_dict							,					size=42							,					max_size=84							,					pad_and_return_pixel_mask=UpperCAmelCase    )
													self.assertEqual(image_processor.size							,					{"shortest_edge": 42, "longest_edge": 84}    )
													self.assertEqual(image_processor.do_pad							,					UpperCAmelCase    )






						def       lowerCamelCase__     (       self      :			List[Any]    ):
													pass






						def       lowerCamelCase__     (       self      :			Union[str, Any]    ):
													# Initialize image_processing
													__lowerCamelCase					:		int		   =     self.image_processing_class(**self.image_processor_dict    )
													# create random PIL images
													__lowerCamelCase					:		Any		   =     prepare_image_inputs(self.image_processor_tester							,					equal_resolution=UpperCAmelCase    )
													for image in image_inputs:
																				self.assertIsInstance(UpperCAmelCase							,					Image.Image    )

													# Test not batched input
													__lowerCamelCase					:		Optional[Any]		   =     image_processing(image_inputs[0]							,					return_tensors="pt"    ).pixel_values

													__lowerCamelCase   ,  __lowerCamelCase					:		List[str]		   =     self.image_processor_tester.get_expected_values(UpperCAmelCase    )

													self.assertEqual(
													    encoded_images.shape							,					(1, self.image_processor_tester.num_channels, expected_height, expected_width)							,					)

													# Test batched
													__lowerCamelCase   ,  __lowerCamelCase					:		Optional[Any]		   =     self.image_processor_tester.get_expected_values(UpperCAmelCase							,					batched=UpperCAmelCase    )

													__lowerCamelCase					:		str		   =     image_processing(UpperCAmelCase							,					return_tensors="pt"    ).pixel_values
													self.assertEqual(
													    encoded_images.shape							,					(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )							,					)






						def       lowerCamelCase__     (       self      :			Any    ):
													# Initialize image_processing
													__lowerCamelCase					:		Union[str, Any]		   =     self.image_processing_class(**self.image_processor_dict    )
													# create random numpy tensors
													__lowerCamelCase					:		List[Any]		   =     prepare_image_inputs(self.image_processor_tester							,					equal_resolution=UpperCAmelCase							,					numpify=UpperCAmelCase    )
													for image in image_inputs:
																				self.assertIsInstance(UpperCAmelCase							,					np.ndarray    )

													# Test not batched input
													__lowerCamelCase					:		Tuple		   =     image_processing(image_inputs[0]							,					return_tensors="pt"    ).pixel_values

													__lowerCamelCase   ,  __lowerCamelCase					:		Dict		   =     self.image_processor_tester.get_expected_values(UpperCAmelCase    )

													self.assertEqual(
													    encoded_images.shape							,					(1, self.image_processor_tester.num_channels, expected_height, expected_width)							,					)

													# Test batched
													__lowerCamelCase					:		Dict		   =     image_processing(UpperCAmelCase							,					return_tensors="pt"    ).pixel_values

													__lowerCamelCase   ,  __lowerCamelCase					:		Tuple		   =     self.image_processor_tester.get_expected_values(UpperCAmelCase							,					batched=UpperCAmelCase    )

													self.assertEqual(
													    encoded_images.shape							,					(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )							,					)






						def       lowerCamelCase__     (       self      :			Optional[int]    ):
													# Initialize image_processing
													__lowerCamelCase					:		int		   =     self.image_processing_class(**self.image_processor_dict    )
													# create random PyTorch tensors
													__lowerCamelCase					:		Dict		   =     prepare_image_inputs(self.image_processor_tester							,					equal_resolution=UpperCAmelCase							,					torchify=UpperCAmelCase    )
													for image in image_inputs:
																				self.assertIsInstance(UpperCAmelCase							,					torch.Tensor    )

													# Test not batched input
													__lowerCamelCase					:		Optional[int]		   =     image_processing(image_inputs[0]							,					return_tensors="pt"    ).pixel_values

													__lowerCamelCase   ,  __lowerCamelCase					:		Tuple		   =     self.image_processor_tester.get_expected_values(UpperCAmelCase    )

													self.assertEqual(
													    encoded_images.shape							,					(1, self.image_processor_tester.num_channels, expected_height, expected_width)							,					)

													# Test batched
													__lowerCamelCase					:		Dict		   =     image_processing(UpperCAmelCase							,					return_tensors="pt"    ).pixel_values

													__lowerCamelCase   ,  __lowerCamelCase					:		Any		   =     self.image_processor_tester.get_expected_values(UpperCAmelCase							,					batched=UpperCAmelCase    )

													self.assertEqual(
													    encoded_images.shape							,					(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )							,					)






						@slow
						def       lowerCamelCase__     (       self      :			List[Any]    ):
													# prepare image and target
													__lowerCamelCase					:		Tuple		   =     Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"    )
													with open("./tests/fixtures/tests_samples/COCO/coco_annotations.txt"							,					"r"    ) as f:
																				__lowerCamelCase					:		str		   =     json.loads(f.read()    )

													__lowerCamelCase					:		List[str]		   =     {"image_id": 39769, "annotations": target}

													# encode them
													__lowerCamelCase					:		Tuple		   =     DetrImageProcessor.from_pretrained("facebook/detr-resnet-50"    )
													__lowerCamelCase					:		str		   =     image_processing(images=UpperCAmelCase							,					annotations=UpperCAmelCase							,					return_tensors="pt"    )

													# verify pixel values
													__lowerCamelCase					:		Tuple		   =     torch.Size([1, 3, 800, 1066]    )
													self.assertEqual(encoding["pixel_values"].shape							,					UpperCAmelCase    )

													__lowerCamelCase					:		str		   =     torch.tensor([0.2_7_9_6, 0.3_1_3_8, 0.3_4_8_1]    )
													self.assertTrue(torch.allclose(encoding["pixel_values"][0, 0, 0, :3]							,					UpperCAmelCase							,					atol=1E-4    )    )

													# verify area
													__lowerCamelCase					:		Optional[Any]		   =     torch.tensor([5_8_8_7.9_6_0_0, 1_1_2_5_0.2_0_6_1, 4_8_9_3_5_3.8_4_3_8, 8_3_7_1_2_2.7_5_0_0, 1_4_7_9_6_7.5_1_5_6, 1_6_5_7_3_2.3_4_3_8]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["area"]							,					UpperCAmelCase    )    )
													# verify boxes
													__lowerCamelCase					:		int		   =     torch.Size([6, 4]    )
													self.assertEqual(encoding["labels"][0]["boxes"].shape							,					UpperCAmelCase    )
													__lowerCamelCase					:		Optional[int]		   =     torch.tensor([0.5_5_0_3, 0.2_7_6_5, 0.0_6_0_4, 0.2_2_1_5]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["boxes"][0]							,					UpperCAmelCase							,					atol=1E-3    )    )
													# verify image_id
													__lowerCamelCase					:		List[str]		   =     torch.tensor([39769]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["image_id"]							,					UpperCAmelCase    )    )
													# verify is_crowd
													__lowerCamelCase					:		Optional[Any]		   =     torch.tensor([0, 0, 0, 0, 0, 0]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["iscrowd"]							,					UpperCAmelCase    )    )
													# verify class_labels
													__lowerCamelCase					:		str		   =     torch.tensor([75, 75, 63, 65, 17, 17]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["class_labels"]							,					UpperCAmelCase    )    )
													# verify orig_size
													__lowerCamelCase					:		Any		   =     torch.tensor([480, 640]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["orig_size"]							,					UpperCAmelCase    )    )
													# verify size
													__lowerCamelCase					:		str		   =     torch.tensor([800, 1066]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["size"]							,					UpperCAmelCase    )    )







						@slow
						def       lowerCamelCase__     (       self      :			Union[str, Any]    ):
													# prepare image, target and masks_path
													__lowerCamelCase					:		Union[str, Any]		   =     Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"    )
													with open("./tests/fixtures/tests_samples/COCO/coco_panoptic_annotations.txt"							,					"r"    ) as f:
																				__lowerCamelCase					:		Optional[int]		   =     json.loads(f.read()    )

													__lowerCamelCase					:		str		   =     {"file_name": "000000039769.png", "image_id": 39769, "segments_info": target}

													__lowerCamelCase					:		Optional[Any]		   =     pathlib.Path("./tests/fixtures/tests_samples/COCO/coco_panoptic"    )

													# encode them
													__lowerCamelCase					:		List[str]		   =     DetrImageProcessor.from_pretrained("facebook/detr-resnet-50-panoptic"    )
													__lowerCamelCase					:		List[str]		   =     image_processing(images=UpperCAmelCase							,					annotations=UpperCAmelCase							,					masks_path=UpperCAmelCase							,					return_tensors="pt"    )

													# verify pixel values
													__lowerCamelCase					:		Optional[Any]		   =     torch.Size([1, 3, 800, 1066]    )
													self.assertEqual(encoding["pixel_values"].shape							,					UpperCAmelCase    )

													__lowerCamelCase					:		Optional[Any]		   =     torch.tensor([0.2_7_9_6, 0.3_1_3_8, 0.3_4_8_1]    )
													self.assertTrue(torch.allclose(encoding["pixel_values"][0, 0, 0, :3]							,					UpperCAmelCase							,					atol=1E-4    )    )

													# verify area
													__lowerCamelCase					:		Dict		   =     torch.tensor([1_4_7_9_7_9.6_8_7_5, 1_6_5_5_2_7.0_4_6_9, 4_8_4_6_3_8.5_9_3_8, 1_1_2_9_2.9_3_7_5, 5_8_7_9.6_5_6_2, 7_6_3_4.1_1_4_7]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["area"]							,					UpperCAmelCase    )    )
													# verify boxes
													__lowerCamelCase					:		Optional[int]		   =     torch.Size([6, 4]    )
													self.assertEqual(encoding["labels"][0]["boxes"].shape							,					UpperCAmelCase    )
													__lowerCamelCase					:		Tuple		   =     torch.tensor([0.2_6_2_5, 0.5_4_3_7, 0.4_6_8_8, 0.8_6_2_5]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["boxes"][0]							,					UpperCAmelCase							,					atol=1E-3    )    )
													# verify image_id
													__lowerCamelCase					:		Dict		   =     torch.tensor([39769]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["image_id"]							,					UpperCAmelCase    )    )
													# verify is_crowd
													__lowerCamelCase					:		Tuple		   =     torch.tensor([0, 0, 0, 0, 0, 0]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["iscrowd"]							,					UpperCAmelCase    )    )
													# verify class_labels
													__lowerCamelCase					:		List[str]		   =     torch.tensor([17, 17, 63, 75, 75, 93]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["class_labels"]							,					UpperCAmelCase    )    )
													# verify masks
													__lowerCamelCase					:		int		   =     822873
													self.assertEqual(encoding["labels"][0]["masks"].sum().item()							,					UpperCAmelCase    )
													# verify orig_size
													__lowerCamelCase					:		Any		   =     torch.tensor([480, 640]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["orig_size"]							,					UpperCAmelCase    )    )
													# verify size
													__lowerCamelCase					:		List[str]		   =     torch.tensor([800, 1066]    )
													self.assertTrue(torch.allclose(encoding["labels"][0]["size"]							,					UpperCAmelCase    )    )
"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A  =				logging.get_logger(__name__)

__A  =				{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A  =				[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__A  =				{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__A  =				{F"""funnel-transformer/{name}""": 512 for name in _model_names}
__A  =				{F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}
class  _snake_case     (					a__		):
						snake_case__			  =			VOCAB_FILES_NAMES
						snake_case__			  =			PRETRAINED_VOCAB_FILES_MAP
						snake_case__			  =			PRETRAINED_INIT_CONFIGURATION
						snake_case__			  =			FunnelTokenizer
						snake_case__			  =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						snake_case__			  =			2






						def __init__(       self      :			List[Any]							,					UpperCAmelCase      :			List[str]=None							,					UpperCAmelCase      :			Optional[int]=None							,					UpperCAmelCase      :			List[Any]=True							,					UpperCAmelCase      :			int="<unk>"							,					UpperCAmelCase      :			List[Any]="<sep>"							,					UpperCAmelCase      :			List[str]="<pad>"							,					UpperCAmelCase      :			Union[str, Any]="<cls>"							,					UpperCAmelCase      :			int="<mask>"							,					UpperCAmelCase      :			List[str]="<s>"							,					UpperCAmelCase      :			Union[str, Any]="</s>"							,					UpperCAmelCase      :			List[Any]=True							,					UpperCAmelCase      :			Tuple=True							,					UpperCAmelCase      :			int=None							,					UpperCAmelCase      :			int="##"							,					**UpperCAmelCase      :			Dict							,					):
													super().__init__(
													    UpperCAmelCase							,					tokenizer_file=UpperCAmelCase							,					do_lower_case=UpperCAmelCase							,					unk_token=UpperCAmelCase							,					sep_token=UpperCAmelCase							,					pad_token=UpperCAmelCase							,					cls_token=UpperCAmelCase							,					mask_token=UpperCAmelCase							,					bos_token=UpperCAmelCase							,					eos_token=UpperCAmelCase							,					clean_text=UpperCAmelCase							,					tokenize_chinese_chars=UpperCAmelCase							,					strip_accents=UpperCAmelCase							,					wordpieces_prefix=UpperCAmelCase							,					**UpperCAmelCase							,					)

													__lowerCamelCase					:		List[Any]		   =     json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
													if (
													    normalizer_state.get("lowercase"							,					UpperCAmelCase    ) != do_lower_case
													    or normalizer_state.get("strip_accents"							,					UpperCAmelCase    ) != strip_accents
													    or normalizer_state.get("handle_chinese_chars"							,					UpperCAmelCase    ) != tokenize_chinese_chars
													):
																				__lowerCamelCase					:		Tuple		   =     getattr(UpperCAmelCase							,					normalizer_state.pop("type"    )    )
																				__lowerCamelCase					:		Optional[int]		   =     do_lower_case
																				__lowerCamelCase					:		Union[str, Any]		   =     strip_accents
																				__lowerCamelCase					:		Optional[Any]		   =     tokenize_chinese_chars
																				__lowerCamelCase					:		Optional[Any]		   =     normalizer_class(**UpperCAmelCase    )

													__lowerCamelCase					:		List[Any]		   =     do_lower_case






						def       lowerCamelCase__     (       self      :			Union[str, Any]							,					UpperCAmelCase      :			Union[str, Any]							,					UpperCAmelCase      :			Tuple=None    ):
													__lowerCamelCase					:		List[Any]		   =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

													if token_ids_a:
																				output += token_ids_a + [self.sep_token_id]

													return output






						def       lowerCamelCase__     (       self      :			Dict							,					UpperCAmelCase      :			List[int]							,					UpperCAmelCase      :			Optional[List[int]] = None    ):
													__lowerCamelCase					:		Tuple		   =     [self.sep_token_id]
													__lowerCamelCase					:		int		   =     [self.cls_token_id]
													if token_ids_a is None:
																				return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0]
													return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







						def       lowerCamelCase__     (       self      :			Tuple							,					UpperCAmelCase      :			str							,					UpperCAmelCase      :			Optional[str] = None    ):
													__lowerCamelCase					:		List[Any]		   =     self._tokenizer.model.save(UpperCAmelCase							,					name=UpperCAmelCase    )
													return tuple(UpperCAmelCase    )