





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_							=				logging.get_logger(__name__)


UpperCAmelCase_							=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCAmelCase_							=				{
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

UpperCAmelCase_							=				{
    """facebook/bart-base""": 1_024,
    """facebook/bart-large""": 1_024,
    """facebook/bart-large-mnli""": 1_024,
    """facebook/bart-large-cnn""": 1_024,
    """facebook/bart-large-xsum""": 1_024,
    """yjernite/bart_eli5""": 1_024,
}





@lru_cache()
def  __magic_name__  (				)				-> str:

							"""simple docstring"""



							lowercase_	:   int  				=     (
							    list(range(ord("""!"""				)				,       ord("""~"""				) + 1				)				) + list(range(ord("""¡"""				)				,       ord("""¬"""				) + 1				)				) + list(range(ord("""®"""				)				,       ord("""ÿ"""				) + 1				)				)
							)
							lowercase_	:   int  				=     bs[:]
							lowercase_	:   Union[str, Any]  				=     0
							for b in range(2**8				):
														if b not in bs:
																					bs.append(lowercase				)
																					cs.append(2**8 + n				)
																					n += 1
							lowercase_	:   Optional[int]  				=     [chr(lowercase				) for n in cs]
							return dict(zip(lowercase				,       lowercase				)				)





def  __magic_name__  (				lowercase				)				-> int:

							"""simple docstring"""



							lowercase_	:   List[Any]  				=     set()
							lowercase_	:   Dict  				=     word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)				)
														lowercase_	:   Optional[int]  				=     char
							return pairs




class     UpperCamelCase__					(     lowerCamelCase__				):






		'''simple docstring'''






		__a       :       Tuple       =	VOCAB_FILES_NAMES
		__a       :       Optional[int]       =	PRETRAINED_VOCAB_FILES_MAP
		__a       :       int       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__a       :       str       =	["""input_ids""", """attention_mask"""]






		def __init__(							self,   snake_case__,   snake_case__,   snake_case__="replace",   snake_case__="<s>",   snake_case__="</s>",   snake_case__="</s>",   snake_case__="<s>",   snake_case__="<unk>",   snake_case__="<pad>",   snake_case__="<mask>",   snake_case__=False,   **snake_case__,   )       -> List[Any]:






									"""simple docstring"""
									lowercase_	:   Optional[Any]  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else bos_token
									lowercase_	:   Optional[Any]  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else eos_token
									lowercase_	:   List[Any]  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else sep_token
									lowercase_	:   Optional[int]  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else cls_token
									lowercase_	:   Optional[Any]  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else unk_token
									lowercase_	:   Tuple  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									lowercase_	:   Any  				=     AddedToken(snake_case__,   lstrip=snake_case__,   rstrip=snake_case__  ) if isinstance(snake_case__,   snake_case__  ) else mask_token

									super().__init__(
									    errors=snake_case__,   bos_token=snake_case__,   eos_token=snake_case__,   unk_token=snake_case__,   sep_token=snake_case__,   cls_token=snake_case__,   pad_token=snake_case__,   mask_token=snake_case__,   add_prefix_space=snake_case__,   **snake_case__,   )

									with open(snake_case__,   encoding="""utf-8"""  ) as vocab_handle:
																lowercase_	:   Union[str, Any]  				=     json.load(snake_case__  )
									lowercase_	:   Optional[int]  				=     {v: k for k, v in self.encoder.items()}
									lowercase_	:   int  				=     errors  # how to handle errors in decoding
									lowercase_	:   List[str]  				=     bytes_to_unicode()
									lowercase_	:   str  				=     {v: k for k, v in self.byte_encoder.items()}
									with open(snake_case__,   encoding="""utf-8"""  ) as merges_handle:
																lowercase_	:   Any  				=     merges_handle.read().split("""\n"""  )[1:-1]
									lowercase_	:   Any  				=     [tuple(merge.split()  ) for merge in bpe_merges]
									lowercase_	:   Tuple  				=     dict(zip(snake_case__,   range(len(snake_case__  )  )  )  )
									lowercase_	:   Dict  				=     {}
									lowercase_	:   Any  				=     add_prefix_space

									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
									lowercase_	:   Optional[int]  				=     re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )






		@property
		def 						snake_case__				(							self  )       -> Any:






									"""simple docstring"""
									return len(self.encoder  )






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									return dict(self.encoder,   **self.added_tokens_encoder  )






		def 						snake_case__				(							self,   snake_case__  )       -> List[str]:






									"""simple docstring"""
									if token in self.cache:
																return self.cache[token]
									lowercase_	:   Optional[int]  				=     tuple(snake_case__  )
									lowercase_	:   List[str]  				=     get_pairs(snake_case__  )

									if not pairs:
																return token

									while True:
																lowercase_	:   Dict  				=     min(snake_case__,   key=lambda snake_case__  : self.bpe_ranks.get(snake_case__,   float("""inf"""  )  )  )
																if bigram not in self.bpe_ranks:
																							break
																lowercase_				,		lowercase_	:   int  				=     bigram
																lowercase_	:   Dict  				=     []
																lowercase_	:   str  				=     0
																while i < len(snake_case__  ):
																							try:
																														lowercase_	:   List[Any]  				=     word.index(snake_case__,   snake_case__  )
																							except ValueError:
																														new_word.extend(word[i:]  )
																														break
																							else:
																														new_word.extend(word[i:j]  )
																														lowercase_	:   int  				=     j

																							if word[i] == first and i < len(snake_case__  ) - 1 and word[i + 1] == second:
																														new_word.append(first + second  )
																														i += 2
																							else:
																														new_word.append(word[i]  )
																														i += 1
																lowercase_	:   Dict  				=     tuple(snake_case__  )
																lowercase_	:   List[Any]  				=     new_word
																if len(snake_case__  ) == 1:
																							break
																else:
																							lowercase_	:   Tuple  				=     get_pairs(snake_case__  )
									lowercase_	:   Optional[int]  				=     """ """.join(snake_case__  )
									lowercase_	:   Optional[Any]  				=     word
									return word






		def 						snake_case__				(							self,   snake_case__  )       -> Union[str, Any]:






									"""simple docstring"""
									lowercase_	:   List[str]  				=     []
									for token in re.findall(self.pat,   snake_case__  ):
																lowercase_	:   Tuple  				=     """""".join(
																    self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__  ).split(""" """  )  )
									return bpe_tokens






		def 						snake_case__				(							self,   snake_case__  )       -> List[Any]:






									"""simple docstring"""
									return self.encoder.get(snake_case__,   self.encoder.get(self.unk_token  )  )






		def 						snake_case__				(							self,   snake_case__  )       -> List[str]:






									"""simple docstring"""
									return self.decoder.get(snake_case__  )






		def 						snake_case__				(							self,   snake_case__  )       -> Optional[int]:






									"""simple docstring"""
									lowercase_	:   List[str]  				=     """""".join(snake_case__  )
									lowercase_	:   Optional[Any]  				=     bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8""",   errors=self.errors  )
									return text






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None  )       -> Tuple[str]:






									"""simple docstring"""
									if not os.path.isdir(snake_case__  ):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																return
									lowercase_	:   Optional[int]  				=     os.path.join(
									    snake_case__,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
									lowercase_	:   int  				=     os.path.join(
									    snake_case__,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

									with open(snake_case__,   """w""",   encoding="""utf-8"""  ) as f:
																f.write(json.dumps(self.encoder,   indent=2,   sort_keys=snake_case__,   ensure_ascii=snake_case__  ) + """\n"""  )

									lowercase_	:   List[Any]  				=     0
									with open(snake_case__,   """w""",   encoding="""utf-8"""  ) as writer:
																writer.write("""#version: 0.2\n"""  )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),   key=lambda snake_case__  : kv[1]  ):
																							if index != token_index:
																														logger.warning(
																														    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																														    """ Please check that the tokenizer is not corrupted!"""  )
																														lowercase_	:   List[Any]  				=     token_index
																							writer.write(""" """.join(snake_case__  ) + """\n"""  )
																							index += 1

									return vocab_file, merge_file






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None  )       -> List[int]:






									"""simple docstring"""
									if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									lowercase_	:   Optional[int]  				=     [self.cls_token_id]
									lowercase_	:   List[Any]  				=     [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None,   snake_case__ = False  )       -> List[int]:






									"""simple docstring"""
									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=snake_case__,   token_ids_a=snake_case__,   already_has_special_tokens=snake_case__  )

									if token_ids_a is None:
																return [1] + ([0] * len(snake_case__  )) + [1]
									return [1] + ([0] * len(snake_case__  )) + [1, 1] + ([0] * len(snake_case__  )) + [1]






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None  )       -> List[int]:






									"""simple docstring"""
									lowercase_	:   List[str]  				=     [self.sep_token_id]
									lowercase_	:   Optional[Any]  				=     [self.cls_token_id]

									if token_ids_a is None:
																return len(cls + token_ids_a + sep  ) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]






		def 						snake_case__				(							self,   snake_case__,   snake_case__=False,   **snake_case__  )       -> Optional[int]:






									"""simple docstring"""
									lowercase_	:   Tuple  				=     kwargs.pop("""add_prefix_space""",   self.add_prefix_space  )
									if (is_split_into_words or add_prefix_space) and (len(snake_case__  ) > 0 and not text[0].isspace()):
																lowercase_	:   Tuple  				=     """ """ + text
									return (text, kwargs)






from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
							import PIL

UpperCAmelCase_							=				logging.get_logger(__name__)





def  __magic_name__  (				lowercase				)				-> List[List[ImageInput]]:

							"""simple docstring"""



							if isinstance(lowercase				,       (list, tuple)				) and isinstance(videos[0]				,       (list, tuple)				) and is_valid_image(videos[0][0]				):
														return videos

							elif isinstance(lowercase				,       (list, tuple)				) and is_valid_image(videos[0]				):
														return [videos]

							elif is_valid_image(lowercase				):
														return [[videos]]

							raise ValueError(f"""Could not make batched video from {videos}"""				)




class     UpperCamelCase__					(     lowerCamelCase__				):






		'''simple docstring'''






		__a       :       List[Any]       =	["""pixel_values"""]






		def __init__(							self,   snake_case__ = True,   snake_case__ = None,   snake_case__ = PILImageResampling.BILINEAR,   snake_case__ = True,   snake_case__ = None,   snake_case__ = True,   snake_case__ = 1 / 2_55,   snake_case__ = True,   snake_case__ = True,   snake_case__ = None,   snake_case__ = None,   **snake_case__,   )       -> None:






									"""simple docstring"""
									super().__init__(**snake_case__  )
									lowercase_	:   str  				=     size if size is not None else {"""shortest_edge""": 2_56}
									lowercase_	:   int  				=     get_size_dict(snake_case__,   default_to_square=snake_case__  )
									lowercase_	:   int  				=     crop_size if crop_size is not None else {"""height""": 2_24, """width""": 2_24}
									lowercase_	:   Optional[int]  				=     get_size_dict(snake_case__,   param_name="""crop_size"""  )

									lowercase_	:   List[Any]  				=     do_resize
									lowercase_	:   int  				=     size
									lowercase_	:   int  				=     do_center_crop
									lowercase_	:   Optional[Any]  				=     crop_size
									lowercase_	:   str  				=     resample
									lowercase_	:   Any  				=     do_rescale
									lowercase_	:   Dict  				=     rescale_factor
									lowercase_	:   List[Any]  				=     offset
									lowercase_	:   int  				=     do_normalize
									lowercase_	:   str  				=     image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
									lowercase_	:   Tuple  				=     image_std if image_std is not None else IMAGENET_STANDARD_STD






		def 						snake_case__				(							self,   snake_case__,   snake_case__,   snake_case__ = PILImageResampling.BILINEAR,   snake_case__ = None,   **snake_case__,   )       -> np.ndarray:






									"""simple docstring"""
									lowercase_	:   int  				=     get_size_dict(snake_case__,   default_to_square=snake_case__  )
									if "shortest_edge" in size:
																lowercase_	:   Union[str, Any]  				=     get_resize_output_image_size(snake_case__,   size["""shortest_edge"""],   default_to_square=snake_case__  )
									elif "height" in size and "width" in size:
																lowercase_	:   Dict  				=     (size["""height"""], size["""width"""])
									else:
																raise ValueError(f"""Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"""  )
									return resize(snake_case__,   size=snake_case__,   resample=snake_case__,   data_format=snake_case__,   **snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__,   snake_case__ = None,   **snake_case__,   )       -> np.ndarray:






									"""simple docstring"""
									lowercase_	:   List[Any]  				=     get_size_dict(snake_case__  )
									if "height" not in size or "width" not in size:
																raise ValueError(f"""Size must have 'height' and 'width' as keys. Got {size.keys()}"""  )
									return center_crop(snake_case__,   size=(size["""height"""], size["""width"""]),   data_format=snake_case__,   **snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__,   snake_case__ = True,   snake_case__ = None,   **snake_case__,   )       -> str:






									"""simple docstring"""
									lowercase_	:   Dict  				=     image.astype(np.floataa  )
									if offset:
																lowercase_	:   Optional[Any]  				=     image - (scale / 2)
									return rescale(snake_case__,   scale=snake_case__,   data_format=snake_case__,   **snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__,   snake_case__,   snake_case__ = None,   **snake_case__,   )       -> np.ndarray:






									"""simple docstring"""
									return normalize(snake_case__,   mean=snake_case__,   std=snake_case__,   data_format=snake_case__,   **snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = ChannelDimension.FIRST,   )       -> np.ndarray:






									"""simple docstring"""
									if do_resize and size is None or resample is None:
																raise ValueError("""Size and resample must be specified if do_resize is True."""  )

									if do_center_crop and crop_size is None:
																raise ValueError("""Crop size must be specified if do_center_crop is True."""  )

									if do_rescale and rescale_factor is None:
																raise ValueError("""Rescale factor must be specified if do_rescale is True."""  )

									if do_normalize and (image_mean is None or image_std is None):
																raise ValueError("""Image mean and std must be specified if do_normalize is True."""  )

									if offset and not do_rescale:
																raise ValueError("""For offset, do_rescale must also be set to True."""  )

									# All transformations expect numpy arrays.
									lowercase_	:   Optional[int]  				=     to_numpy_array(snake_case__  )

									if do_resize:
																lowercase_	:   Dict  				=     self.resize(image=snake_case__,   size=snake_case__,   resample=snake_case__  )

									if do_center_crop:
																lowercase_	:   Optional[int]  				=     self.center_crop(snake_case__,   size=snake_case__  )

									if do_rescale:
																lowercase_	:   Dict  				=     self.rescale(image=snake_case__,   scale=snake_case__,   offset=snake_case__  )

									if do_normalize:
																lowercase_	:   Optional[Any]  				=     self.normalize(image=snake_case__,   mean=snake_case__,   std=snake_case__  )

									lowercase_	:   Tuple  				=     to_channel_dimension_format(snake_case__,   snake_case__  )
									return image






		def 						snake_case__				(							self,   snake_case__,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = None,   snake_case__ = ChannelDimension.FIRST,   **snake_case__,   )       -> PIL.Image.Image:






									"""simple docstring"""
									lowercase_	:   Any  				=     do_resize if do_resize is not None else self.do_resize
									lowercase_	:   str  				=     resample if resample is not None else self.resample
									lowercase_	:   List[str]  				=     do_center_crop if do_center_crop is not None else self.do_center_crop
									lowercase_	:   Optional[Any]  				=     do_rescale if do_rescale is not None else self.do_rescale
									lowercase_	:   str  				=     rescale_factor if rescale_factor is not None else self.rescale_factor
									lowercase_	:   List[Any]  				=     offset if offset is not None else self.offset
									lowercase_	:   Optional[Any]  				=     do_normalize if do_normalize is not None else self.do_normalize
									lowercase_	:   Any  				=     image_mean if image_mean is not None else self.image_mean
									lowercase_	:   Optional[Any]  				=     image_std if image_std is not None else self.image_std

									lowercase_	:   Optional[Any]  				=     size if size is not None else self.size
									lowercase_	:   Dict  				=     get_size_dict(snake_case__,   default_to_square=snake_case__  )
									lowercase_	:   Union[str, Any]  				=     crop_size if crop_size is not None else self.crop_size
									lowercase_	:   Tuple  				=     get_size_dict(snake_case__,   param_name="""crop_size"""  )

									if not valid_images(snake_case__  ):
																raise ValueError(
																    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
																    """torch.Tensor, tf.Tensor or jax.ndarray."""  )

									lowercase_	:   Optional[Any]  				=     make_batched(snake_case__  )

									lowercase_	:   Optional[Any]  				=     [
									    [
									        self._preprocess_image(
									            image=snake_case__,   do_resize=snake_case__,   size=snake_case__,   resample=snake_case__,   do_center_crop=snake_case__,   crop_size=snake_case__,   do_rescale=snake_case__,   rescale_factor=snake_case__,   offset=snake_case__,   do_normalize=snake_case__,   image_mean=snake_case__,   image_std=snake_case__,   data_format=snake_case__,   )
									        for img in video
									    ]
									    for video in videos
									]

									lowercase_	:   List[str]  				=     {"""pixel_values""": videos}
									return BatchFeature(data=snake_case__,   tensor_type=snake_case__  )