



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_lowercase     							=						logging.get_logger(__name__)


_lowercase     							=						{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
_lowercase     							=						{
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

_lowercase     							=						{
    '''facebook/bart-base''': 10_24,
    '''facebook/bart-large''': 10_24,
    '''facebook/bart-large-mnli''': 10_24,
    '''facebook/bart-large-cnn''': 10_24,
    '''facebook/bart-large-xsum''': 10_24,
    '''yjernite/bart_eli5''': 10_24,
}
class 				__A					( A_      ):
			UpperCamelCase					:Optional[Any]     =			VOCAB_FILES_NAMES
			UpperCamelCase					:Dict     =			PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase					:List[Any]     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase					:List[Any]     =			['''input_ids''', '''attention_mask''']
			UpperCamelCase					:Optional[Any]     =			BartTokenizer





			def __init__(self							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__=None							,    __magic_name__="replace"							,    __magic_name__="<s>"							,    __magic_name__="</s>"							,    __magic_name__="</s>"							,    __magic_name__="<s>"							,    __magic_name__="<unk>"							,    __magic_name__="<pad>"							,    __magic_name__="<mask>"							,    __magic_name__=False							,    __magic_name__=True							,    **__magic_name__							,    ):
						super().__init__(
						    __UpperCamelCase							,    __UpperCamelCase							,    tokenizer_file=__UpperCamelCase							,    errors=__UpperCamelCase							,    bos_token=__UpperCamelCase							,    eos_token=__UpperCamelCase							,    sep_token=__UpperCamelCase							,    cls_token=__UpperCamelCase							,    unk_token=__UpperCamelCase							,    pad_token=__UpperCamelCase							,    mask_token=__UpperCamelCase							,    add_prefix_space=__UpperCamelCase							,    trim_offsets=__UpperCamelCase							,    **__UpperCamelCase							,    )

						lowerCamelCase__ :      int								=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
						if pre_tok_state.get("""add_prefix_space"""							,    __UpperCamelCase				) != add_prefix_space:
									lowerCamelCase__ :      Dict								=		getattr(__UpperCamelCase							,    pre_tok_state.pop("""type"""				)				)
									lowerCamelCase__ :      int								=		add_prefix_space
									lowerCamelCase__ :      int								=		pre_tok_class(**__UpperCamelCase				)

						lowerCamelCase__ :      List[str]								=		add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						lowerCamelCase__ :      Any								=		"""post_processor"""
						lowerCamelCase__ :      List[str]								=		getattr(self.backend_tokenizer							,    __UpperCamelCase							,    __UpperCamelCase				)
						if tokenizer_component_instance:
									lowerCamelCase__ :      Tuple								=		json.loads(tokenizer_component_instance.__getstate__()				)

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
												lowerCamelCase__ :      Tuple								=		tuple(state["""sep"""]				)
									if "cls" in state:
												lowerCamelCase__ :      Tuple								=		tuple(state["""cls"""]				)

									lowerCamelCase__ :      str								=		False

									if state.get("""add_prefix_space"""							,    __UpperCamelCase				) != add_prefix_space:
												lowerCamelCase__ :      int								=		add_prefix_space
												lowerCamelCase__ :      Optional[int]								=		True

									if state.get("""trim_offsets"""							,    __UpperCamelCase				) != trim_offsets:
												lowerCamelCase__ :      List[str]								=		trim_offsets
												lowerCamelCase__ :      Optional[Any]								=		True

									if changes_to_apply:
												lowerCamelCase__ :      str								=		getattr(__UpperCamelCase							,    state.pop("""type"""				)				)
												lowerCamelCase__ :      Tuple								=		component_class(**__UpperCamelCase				)
												setattr(self.backend_tokenizer							,    __UpperCamelCase							,    __UpperCamelCase				)





			@property
			def        _snake_case    (self				):
						if self._mask_token is None:
									if self.verbose:
												logger.error("""Using mask_token, but it is not set yet."""				)
									return None
						return str(self._mask_token				)





			@mask_token.setter
			def        _snake_case    (self							,    __magic_name__				):
						lowerCamelCase__ :      List[str]								=		AddedToken(__UpperCamelCase							,    lstrip=__UpperCamelCase							,    rstrip=__UpperCamelCase				) if isinstance(__UpperCamelCase							,    __UpperCamelCase				) else value
						lowerCamelCase__ :      Tuple								=		value





			def        _snake_case    (self							,    *__magic_name__							,    **__magic_name__				):
						lowerCamelCase__ :      List[str]								=		kwargs.get("""is_split_into_words"""							,    __UpperCamelCase				)

						if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    """to use it with pretokenized inputs."""				)

						return super()._batch_encode_plus(*__UpperCamelCase							,    **__UpperCamelCase				)





			def        _snake_case    (self							,    *__magic_name__							,    **__magic_name__				):
						lowerCamelCase__ :      Dict								=		kwargs.get("""is_split_into_words"""							,    __UpperCamelCase				)

						if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    """to use it with pretokenized inputs."""				)

						return super()._encode_plus(*__UpperCamelCase							,    **__UpperCamelCase				)





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__ = None				):
						lowerCamelCase__ :      Union[str, Any]								=		self._tokenizer.model.save(__UpperCamelCase							,    name=__UpperCamelCase				)
						return tuple(__UpperCamelCase				)





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__=None				):
						lowerCamelCase__ :      Optional[int]								=		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
									return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__ = None				):
						lowerCamelCase__ :      Any								=		[self.sep_token_id]
						lowerCamelCase__ :      List[str]								=		[self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep				) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]








import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
				import torch

				from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
				_lowercase     							=						False

if is_vision_available():
				from PIL import Image

				from transformers import PixaStructImageProcessor
class 				__A					( unittest.TestCase      ):





			def __init__(self							,    __magic_name__							,    __magic_name__=7							,    __magic_name__=3							,    __magic_name__=18							,    __magic_name__=30							,    __magic_name__=400							,    __magic_name__=None							,    __magic_name__=True							,    __magic_name__=True							,    __magic_name__=None							,    ):
						lowerCamelCase__ :      Optional[Any]								=		size if size is not None else {"""height""": 20, """width""": 20}
						lowerCamelCase__ :      List[str]								=		parent
						lowerCamelCase__ :      str								=		batch_size
						lowerCamelCase__ :      List[str]								=		num_channels
						lowerCamelCase__ :      Optional[int]								=		image_size
						lowerCamelCase__ :      Union[str, Any]								=		min_resolution
						lowerCamelCase__ :      List[Any]								=		max_resolution
						lowerCamelCase__ :      Optional[int]								=		size
						lowerCamelCase__ :      List[str]								=		do_normalize
						lowerCamelCase__ :      str								=		do_convert_rgb
						lowerCamelCase__ :      str								=		[512, 1024, 2048, 4096]
						lowerCamelCase__ :      Any								=		patch_size if patch_size is not None else {"""height""": 16, """width""": 16}





			def        _snake_case    (self				):
						return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}





			def        _snake_case    (self				):
						lowerCamelCase__ :      List[str]								=		"""https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg"""
						lowerCamelCase__ :      int								=		Image.open(requests.get(__magic_name__							,    stream=__magic_name__				).raw				).convert("""RGB"""				)
						return raw_image







@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11					, reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''					, )
@require_torch
@require_vision
class 				__A					( A_					, unittest.TestCase      ):
			UpperCamelCase					:Union[str, Any]     =			PixaStructImageProcessor if is_vision_available() else None





			def        _snake_case    (self				):
						lowerCamelCase__ :      List[str]								=		PixaStructImageProcessingTester(self				)





			@property
			def        _snake_case    (self				):
						return self.image_processor_tester.prepare_image_processor_dict()





			def        _snake_case    (self				):
						lowerCamelCase__ :      int								=		self.image_processing_class(**self.image_processor_dict				)
						self.assertTrue(hasattr(__magic_name__							,    """do_normalize"""				)				)
						self.assertTrue(hasattr(__magic_name__							,    """do_convert_rgb"""				)				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      Tuple								=		self.image_processor_tester.prepare_dummy_image()

						lowerCamelCase__ :      Tuple								=		self.image_processing_class(**self.image_processor_dict				)
						lowerCamelCase__ :      List[str]								=		2048

						lowerCamelCase__ :      Any								=		image_processor(__magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				)
						self.assertTrue(torch.allclose(inputs.flattened_patches.mean()							,    torch.tensor(0.06_06				)							,    atol=1E-3							,    rtol=1E-3				)				)





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      List[Any]								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PIL images
						lowerCamelCase__ :      Optional[Any]								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    Image.Image				)

						# Test not batched input
						lowerCamelCase__ :      Optional[Any]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      List[str]								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PIL images
						lowerCamelCase__ :      Optional[Any]								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    Image.Image				)

						# Test not batched input
						lowerCamelCase__ :      List[Any]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						lowerCamelCase__ :      List[str]								=		True

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									with self.assertRaises(__magic_name__				):
												lowerCamelCase__ :      Tuple								=		image_processor(
												    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches

									lowerCamelCase__ :      Optional[Any]								=		"""Hello"""

									lowerCamelCase__ :      Dict								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__							,    header_text=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      Tuple								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__							,    header_text=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      Any								=		self.image_processing_class(**self.image_processor_dict				)
						# create random numpy tensors
						lowerCamelCase__ :      Dict								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__							,    numpify=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    np.ndarray				)

						lowerCamelCase__ :      Optional[int]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      List[str]								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      List[str]								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      Optional[int]								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PyTorch tensors
						lowerCamelCase__ :      Dict								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__							,    torchify=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    torch.Tensor				)

						# Test not batched input
						lowerCamelCase__ :      List[str]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      Tuple								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )







@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11					, reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''					, )
@require_torch
@require_vision
class 				__A					( A_					, unittest.TestCase      ):
			UpperCamelCase					:Optional[int]     =			PixaStructImageProcessor if is_vision_available() else None





			def        _snake_case    (self				):
						lowerCamelCase__ :      Optional[Any]								=		PixaStructImageProcessingTester(self							,    num_channels=4				)
						lowerCamelCase__ :      Optional[int]								=		3





			@property
			def        _snake_case    (self				):
						return self.image_processor_tester.prepare_image_processor_dict()





			def        _snake_case    (self				):
						lowerCamelCase__ :      Dict								=		self.image_processing_class(**self.image_processor_dict				)
						self.assertTrue(hasattr(__magic_name__							,    """do_normalize"""				)				)
						self.assertTrue(hasattr(__magic_name__							,    """do_convert_rgb"""				)				)





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      str								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PIL images
						lowerCamelCase__ :      List[Any]								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    Image.Image				)

						# Test not batched input
						lowerCamelCase__ :      List[str]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * (self.image_processor_tester.num_channels - 1)
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      str								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )



