

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__						=							logging.get_logger(__name__)


A__						=							{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
A__						=							{
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

A__						=							{
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}




@lru_cache()
def 					_lowercase     (				)				->	Optional[Any]:




					'''simple docstring'''







					__magic_name__							=    (
					    list(range(ord('!'				)					,ord('~'				) + 1				)				) + list(range(ord('¡'				)					,ord('¬'				) + 1				)				) + list(range(ord('®'				)					,ord('ÿ'				) + 1				)				)
					)
					__magic_name__							=    bs[:]
					__magic_name__							=    0
					for b in range(2**8				):
										if b not in bs:
															bs.append(snake_case_				)
															cs.append(2**8 + n				)
															n += 1
					__magic_name__							=    [chr(snake_case_				) for n in cs]
					return dict(zip(snake_case_					,snake_case_				)				)




def 					_lowercase     (				a_				:					List[Any]				)				->	List[str]:




					'''simple docstring'''







					__magic_name__							=    set()
					__magic_name__							=    word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)				)
										__magic_name__							=    char
					return pairs


class 				__UpperCamelCase				(					UpperCamelCase_			):
	_lowercase  :   Optional[int]       				=  VOCAB_FILES_NAMES
	_lowercase  :   Dict       				=  PRETRAINED_VOCAB_FILES_MAP
	_lowercase  :   List[str]       				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowercase  :   List[str]       				=  ["input_ids", "attention_mask"]







	def __init__(   self: int  , __UpperCamelCase: Tuple  , __UpperCamelCase: Tuple  , __UpperCamelCase: Optional[int]="replace"  , __UpperCamelCase: Union[str, Any]="<s>"  , __UpperCamelCase: str="</s>"  , __UpperCamelCase: List[Any]="</s>"  , __UpperCamelCase: Union[str, Any]="<s>"  , __UpperCamelCase: Optional[int]="<unk>"  , __UpperCamelCase: Optional[int]="<pad>"  , __UpperCamelCase: Optional[Any]="<mask>"  , __UpperCamelCase: Optional[Any]=False  , **__UpperCamelCase: List[Any]  , ):

						'''simple docstring'''





						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else bos_token
						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else eos_token
						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else sep_token
						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else cls_token
						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else unk_token
						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else pad_token

						# Mask token behave like a normal word, i.e. include the space before it
						__magic_name__							=    AddedToken(__A  , lstrip=__A  , rstrip=__A      ) if isinstance(__A  , __A      ) else mask_token

						super().__init__(
						    errors=__A  , bos_token=__A  , eos_token=__A  , unk_token=__A  , sep_token=__A  , cls_token=__A  , pad_token=__A  , mask_token=__A  , add_prefix_space=__A  , **__A  , )

						with open(__A  , encoding='utf-8'      ) as vocab_handle:
											__magic_name__							=    json.load(__A      )
						__magic_name__							=    {v: k for k, v in self.encoder.items()}
						__magic_name__							=    errors  # how to handle errors in decoding
						__magic_name__							=    bytes_to_unicode()
						__magic_name__							=    {v: k for k, v in self.byte_encoder.items()}
						with open(__A  , encoding='utf-8'      ) as merges_handle:
											__magic_name__							=    merges_handle.read().split('\n'      )[1:-1]
						__magic_name__							=    [tuple(merge.split()      ) for merge in bpe_merges]
						__magic_name__							=    dict(zip(__A  , range(len(__A      )      )      )      )
						__magic_name__							=    {}
						__magic_name__							=    add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						__magic_name__							=    re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						return len(self.encoder      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]      ):

						'''simple docstring'''





						return dict(self.encoder  , **self.added_tokens_encoder      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Dict  , __UpperCamelCase: List[Any]      ):

						'''simple docstring'''





						if token in self.cache:
											return self.cache[token]
						__magic_name__							=    tuple(__A      )
						__magic_name__							=    get_pairs(__A      )

						if not pairs:
											return token

						while True:
											__magic_name__							=    min(__A  , key=lambda __UpperCamelCase      : self.bpe_ranks.get(__A  , float('inf'      )      )      )
											if bigram not in self.bpe_ranks:
																break
											__magic_name__							=    bigram
											__magic_name__							=    []
											__magic_name__							=    0
											while i < len(__A      ):
																try:
																					__magic_name__							=    word.index(__A  , __A      )
																except ValueError:
																					new_word.extend(word[i:]      )
																					break
																else:
																					new_word.extend(word[i:j]      )
																					__magic_name__							=    j

																if word[i] == first and i < len(__A      ) - 1 and word[i + 1] == second:
																					new_word.append(first + second      )
																					i += 2
																else:
																					new_word.append(word[i]      )
																					i += 1
											__magic_name__							=    tuple(__A      )
											__magic_name__							=    new_word
											if len(__A      ) == 1:
																break
											else:
																__magic_name__							=    get_pairs(__A      )
						__magic_name__							=    " ".join(__A      )
						__magic_name__							=    word
						return word







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , __UpperCamelCase: Tuple      ):

						'''simple docstring'''





						__magic_name__							=    []
						for token in re.findall(self.pat  , __A      ):
											__magic_name__							=    "".join(
											    self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A      ).split(' '      )      )
						return bpe_tokens







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , __UpperCamelCase: Tuple      ):

						'''simple docstring'''





						return self.encoder.get(__A  , self.encoder.get(self.unk_token      )      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]  , __UpperCamelCase: int      ):

						'''simple docstring'''





						return self.decoder.get(__A      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]  , __UpperCamelCase: int      ):

						'''simple docstring'''





						__magic_name__							=    "".join(__A      )
						__magic_name__							=    bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'  , errors=self.errors      )
						return text







	def 	_SCREAMING_SNAKE_CASE							(   self: int  , __UpperCamelCase: str  , __UpperCamelCase: Optional[str] = None      ):

						'''simple docstring'''





						if not os.path.isdir(__A      ):
											logger.error(F'Vocabulary path ({save_directory}) should be a directory'      )
											return
						__magic_name__							=    os.path.join(
						    __A  , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
						__magic_name__							=    os.path.join(
						    __A  , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

						with open(__A  , 'w'  , encoding='utf-8'      ) as f:
											f.write(json.dumps(self.encoder  , indent=2  , sort_keys=__A  , ensure_ascii=__A      ) + '\n'      )

						__magic_name__							=    0
						with open(__A  , 'w'  , encoding='utf-8'      ) as writer:
											writer.write('#version: 0.2\n'      )
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  , key=lambda __UpperCamelCase      : kv[1]      ):
																if index != token_index:
																					logger.warning(
																					    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																					    ' Please check that the tokenizer is not corrupted!'      )
																					__magic_name__							=    token_index
																writer.write(' '.join(__A      ) + '\n'      )
																index += 1

						return vocab_file, merge_file







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , __UpperCamelCase: List[int]  , __UpperCamelCase: Optional[List[int]] = None      ):

						'''simple docstring'''





						if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__magic_name__							=    [self.cls_token_id]
						__magic_name__							=    [self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]  , __UpperCamelCase: List[int]  , __UpperCamelCase: Optional[List[int]] = None  , __UpperCamelCase: bool = False      ):

						'''simple docstring'''





						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__A  , token_ids_a=__A  , already_has_special_tokens=__A      )

						if token_ids_a is None:
											return [1] + ([0] * len(__A      )) + [1]
						return [1] + ([0] * len(__A      )) + [1, 1] + ([0] * len(__A      )) + [1]







	def 	_SCREAMING_SNAKE_CASE							(   self: Any  , __UpperCamelCase: List[int]  , __UpperCamelCase: Optional[List[int]] = None      ):

						'''simple docstring'''





						__magic_name__							=    [self.sep_token_id]
						__magic_name__							=    [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep      ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: Dict  , __UpperCamelCase: Any=False  , **__UpperCamelCase: Union[str, Any]      ):

						'''simple docstring'''





						__magic_name__							=    kwargs.pop('add_prefix_space'  , self.add_prefix_space      )
						if (is_split_into_words or add_prefix_space) and (len(__A      ) > 0 and not text[0].isspace()):
											__magic_name__							=    " " + text
						return (text, kwargs)







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
						import PIL


A__						=							logging.get_logger(__name__)

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):
	_lowercase  :   Tuple       				=  ["pixel_values"]







	def __init__(   self: Optional[Any]  , __UpperCamelCase: bool = True  , __UpperCamelCase: Dict[str, int] = None  , __UpperCamelCase: int = 0.9  , __UpperCamelCase: PILImageResampling = PILImageResampling.BICUBIC  , __UpperCamelCase: bool = True  , __UpperCamelCase: Dict[str, int] = None  , __UpperCamelCase: Union[int, float] = 1 / 2_55  , __UpperCamelCase: bool = True  , __UpperCamelCase: bool = True  , __UpperCamelCase: Optional[Union[float, List[float]]] = None  , __UpperCamelCase: Optional[Union[float, List[float]]] = None  , **__UpperCamelCase: List[str]  , ):

						'''simple docstring'''





						super().__init__(**__UpperCamelCase      )
						__magic_name__							=    size if size is not None else {'shortest_edge': 2_24}
						__magic_name__							=    get_size_dict(__UpperCamelCase  , default_to_square=__UpperCamelCase      )
						__magic_name__							=    crop_size if crop_size is not None else {'height': 2_24, 'width': 2_24}
						__magic_name__							=    get_size_dict(__UpperCamelCase  , param_name='crop_size'      )

						__magic_name__							=    do_resize
						__magic_name__							=    size
						__magic_name__							=    crop_pct
						__magic_name__							=    resample
						__magic_name__							=    do_center_crop
						__magic_name__							=    crop_size
						__magic_name__							=    do_rescale
						__magic_name__							=    rescale_factor
						__magic_name__							=    do_normalize
						__magic_name__							=    image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
						__magic_name__							=    image_std if image_std is not None else IMAGENET_DEFAULT_STD







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]  , __UpperCamelCase: np.ndarray  , __UpperCamelCase: Dict[str, int]  , __UpperCamelCase: Optional[float] = None  , __UpperCamelCase: PILImageResampling = PILImageResampling.BICUBIC  , __UpperCamelCase: Optional[Union[str, ChannelDimension]] = None  , **__UpperCamelCase: Optional[Any]  , ):

						'''simple docstring'''





						__magic_name__							=    get_size_dict(__UpperCamelCase  , default_to_square=__UpperCamelCase      )
						if "shortest_edge" not in size and ("height" not in size or "width" not in size):
											raise ValueError(F'size must contain \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'      )
						if crop_pct is not None:
											if "shortest_edge" in size:
																__magic_name__							=    int(size['shortest_edge'] / crop_pct      )
											elif "height" in size and "width" in size:
																if size["height"] == size["width"]:
																					__magic_name__							=    int(size['height'] / crop_pct      )
																else:
																					__magic_name__							=    (int(size['height'] / crop_pct      ), int(size['width'] / crop_pct      ))
											else:
																raise ValueError('Invalid size for resize: {}'.format(__UpperCamelCase      )      )

											__magic_name__							=    get_resize_output_image_size(__UpperCamelCase  , size=__UpperCamelCase  , default_to_square=__UpperCamelCase      )
						else:
											if "shortest_edge" in size:
																__magic_name__							=    get_resize_output_image_size(__UpperCamelCase  , size=size['shortest_edge']  , default_to_square=__UpperCamelCase      )
											elif "height" in size and "width" in size:
																__magic_name__							=    (size['height'], size['width'])
											else:
																raise ValueError('Invalid size for resize: {}'.format(__UpperCamelCase      )      )

						return resize(__UpperCamelCase  , size=__UpperCamelCase  , resample=__UpperCamelCase  , data_format=__UpperCamelCase  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Any  , __UpperCamelCase: np.ndarray  , __UpperCamelCase: Dict[str, int]  , __UpperCamelCase: Optional[Union[str, ChannelDimension]] = None  , **__UpperCamelCase: List[str]  , ):

						'''simple docstring'''





						__magic_name__							=    get_size_dict(__UpperCamelCase      )
						if "height" not in size or "width" not in size:
											raise ValueError(F'size must contain \'height\' and \'width\' as keys. Got {size.keys()}'      )
						return center_crop(__UpperCamelCase  , size=(size['height'], size['width'])  , data_format=__UpperCamelCase  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]  , __UpperCamelCase: np.ndarray  , __UpperCamelCase: Union[int, float]  , __UpperCamelCase: Optional[Union[str, ChannelDimension]] = None  , **__UpperCamelCase: Optional[Any]  , ):

						'''simple docstring'''





						return rescale(__UpperCamelCase  , scale=__UpperCamelCase  , data_format=__UpperCamelCase  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Dict  , __UpperCamelCase: np.ndarray  , __UpperCamelCase: Union[float, List[float]]  , __UpperCamelCase: Union[float, List[float]]  , __UpperCamelCase: Optional[Union[str, ChannelDimension]] = None  , **__UpperCamelCase: Optional[int]  , ):

						'''simple docstring'''





						return normalize(__UpperCamelCase  , mean=__UpperCamelCase  , std=__UpperCamelCase  , data_format=__UpperCamelCase  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , __UpperCamelCase: ImageInput  , __UpperCamelCase: bool = None  , __UpperCamelCase: Dict[str, int] = None  , __UpperCamelCase: int = None  , __UpperCamelCase: PILImageResampling = None  , __UpperCamelCase: bool = None  , __UpperCamelCase: Dict[str, int] = None  , __UpperCamelCase: bool = None  , __UpperCamelCase: float = None  , __UpperCamelCase: bool = None  , __UpperCamelCase: Optional[Union[float, List[float]]] = None  , __UpperCamelCase: Optional[Union[float, List[float]]] = None  , __UpperCamelCase: Optional[Union[str, TensorType]] = None  , __UpperCamelCase: ChannelDimension = ChannelDimension.FIRST  , **__UpperCamelCase: Any  , ):

						'''simple docstring'''





						__magic_name__							=    do_resize if do_resize is not None else self.do_resize
						__magic_name__							=    crop_pct if crop_pct is not None else self.crop_pct
						__magic_name__							=    resample if resample is not None else self.resample
						__magic_name__							=    do_center_crop if do_center_crop is not None else self.do_center_crop
						__magic_name__							=    do_rescale if do_rescale is not None else self.do_rescale
						__magic_name__							=    rescale_factor if rescale_factor is not None else self.rescale_factor
						__magic_name__							=    do_normalize if do_normalize is not None else self.do_normalize
						__magic_name__							=    image_mean if image_mean is not None else self.image_mean
						__magic_name__							=    image_std if image_std is not None else self.image_std

						__magic_name__							=    size if size is not None else self.size
						__magic_name__							=    get_size_dict(__UpperCamelCase  , default_to_square=__UpperCamelCase      )
						__magic_name__							=    crop_size if crop_size is not None else self.crop_size
						__magic_name__							=    get_size_dict(__UpperCamelCase  , param_name='crop_size'      )

						__magic_name__							=    make_list_of_images(__UpperCamelCase      )

						if not valid_images(__UpperCamelCase      ):
											raise ValueError(
											    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
											    'torch.Tensor, tf.Tensor or jax.ndarray.'      )

						if do_resize and size is None or resample is None:
											raise ValueError('Size and resample must be specified if do_resize is True.'      )

						if do_center_crop and crop_pct is None:
											raise ValueError('Crop_pct must be specified if do_center_crop is True.'      )

						if do_rescale and rescale_factor is None:
											raise ValueError('Rescale factor must be specified if do_rescale is True.'      )

						if do_normalize and (image_mean is None or image_std is None):
											raise ValueError('Image mean and std must be specified if do_normalize is True.'      )

						# All transformations expect numpy arrays.
						__magic_name__							=    [to_numpy_array(__UpperCamelCase      ) for image in images]

						if do_resize:
											__magic_name__							=    [self.resize(image=__UpperCamelCase  , size=__UpperCamelCase  , crop_pct=__UpperCamelCase  , resample=__UpperCamelCase      ) for image in images]

						if do_center_crop:
											__magic_name__							=    [self.center_crop(image=__UpperCamelCase  , size=__UpperCamelCase      ) for image in images]

						if do_rescale:
											__magic_name__							=    [self.rescale(image=__UpperCamelCase  , scale=__UpperCamelCase      ) for image in images]

						if do_normalize:
											__magic_name__							=    [self.normalize(image=__UpperCamelCase  , mean=__UpperCamelCase  , std=__UpperCamelCase      ) for image in images]

						__magic_name__							=    [to_channel_dimension_format(__UpperCamelCase  , __UpperCamelCase      ) for image in images]

						__magic_name__							=    {'pixel_values': images}
						return BatchFeature(data=__UpperCamelCase  , tensor_type=__UpperCamelCase      )




