






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase :      Optional[Any]										=				logging.get_logger(__name__)


__lowerCamelCase :      Tuple										=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
__lowerCamelCase :      Dict										=				{
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

__lowerCamelCase :      Optional[Any]										=				{
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}







@lru_cache()
def 					SCREAMING_SNAKE_CASE    (			):
	snake_case__			:				Optional[Any]       =  (
	    list(range(ord("!"       )					,  ord("~"       ) + 1       )       ) + list(range(ord("¡"       )					,  ord("¬"       ) + 1       )       ) + list(range(ord("®"       )					,  ord("ÿ"       ) + 1       )       )
	)
	snake_case__			:				int       =  bs[:]
	snake_case__			:				Any       =  0
	for b in range(2**8       ):
		if b not in bs:
			bs.append(snake_case_       )
			cs.append(2**8 + n       )
			n += 1
	snake_case__			:				Optional[Any]       =  [chr(snake_case_       ) for n in cs]
	return dict(zip(snake_case_					,  snake_case_       )       )







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							List[str]       ):
	snake_case__			:				Dict       =  set()
	snake_case__			:				List[Any]       =  word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)       )
		snake_case__			:				Tuple       =  char
	return pairs





class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_						):





					"""simple docstring"""






					a_     =		VOCAB_FILES_NAMES
					a_     =		PRETRAINED_VOCAB_FILES_MAP
					a_     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					a_     =		["input_ids", "attention_mask"]



					def __init__( self					:  Any							,		__A					:  Optional[Any]							,		__A					:  Tuple							,		__A					:  Optional[Any]="replace"							,		__A					:  Tuple="<s>"							,		__A					:  Tuple="</s>"							,		__A					:  Union[str, Any]="</s>"							,		__A					:  Tuple="<s>"							,		__A					:  Optional[Any]="<unk>"							,		__A					:  Optional[Any]="<pad>"							,		__A					:  Tuple="<mask>"							,		__A					:  str=False							,		**__A					:  List[str]							,		):
						snake_case__			:				int       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else bos_token
						snake_case__			:				Union[str, Any]       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else eos_token
						snake_case__			:				Any       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else sep_token
						snake_case__			:				int       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else cls_token
						snake_case__			:				List[str]       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else unk_token
						snake_case__			:				Optional[Any]       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else pad_token

						# Mask token behave like a normal word, i.e. include the space before it
						snake_case__			:				List[str]       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else mask_token

						super().__init__(
						    errors=__A							,		bos_token=__A							,		eos_token=__A							,		unk_token=__A							,		sep_token=__A							,		cls_token=__A							,		pad_token=__A							,		mask_token=__A							,		add_prefix_space=__A							,		**__A							,		)

						with open(__A							,		encoding="utf-8"	) as vocab_handle:
							snake_case__			:				List[Any]       =  json.load(__A	)
						snake_case__			:				str       =  {v: k for k, v in self.encoder.items()}
						snake_case__			:				int       =  errors  # how to handle errors in decoding
						snake_case__			:				List[Any]       =  bytes_to_unicode()
						snake_case__			:				str       =  {v: k for k, v in self.byte_encoder.items()}
						with open(__A							,		encoding="utf-8"	) as merges_handle:
							snake_case__			:				int       =  merges_handle.read().split("\n"	)[1:-1]
						snake_case__			:				Dict       =  [tuple(merge.split()	) for merge in bpe_merges]
						snake_case__			:				Dict       =  dict(zip(__A							,		range(len(__A	)	)	)	)
						snake_case__			:				Any       =  {}
						snake_case__			:				List[str]       =  add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						snake_case__			:				List[Any]       =  re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"	)



					@property
					def 	_lowercase					( self					:  Dict	):
						return len(self.encoder	)



					def 	_lowercase					( self					:  Union[str, Any]	):
						return dict(self.encoder							,		**self.added_tokens_encoder	)



					def 	_lowercase					( self					:  Dict							,		__A					:  Optional[int]	):
						if token in self.cache:
							return self.cache[token]
						snake_case__			:				Tuple       =  tuple(__A	)
						snake_case__			:				int       =  get_pairs(__A	)

						if not pairs:
							return token

						while True:
							snake_case__			:				Optional[Any]       =  min(__A							,		key=lambda __A	: self.bpe_ranks.get(__A							,		float("inf"	)	)	)
							if bigram not in self.bpe_ranks:
								break
							snake_case__,						snake_case__			:				str       =  bigram
							snake_case__			:				str       =  []
							snake_case__			:				int       =  0
							while i < len(__A	):
								try:
									snake_case__			:				Union[str, Any]       =  word.index(__A							,		__A	)
								except ValueError:
									new_word.extend(word[i:]	)
									break
								else:
									new_word.extend(word[i:j]	)
									snake_case__			:				List[str]       =  j

								if word[i] == first and i < len(__A	) - 1 and word[i + 1] == second:
									new_word.append(first + second	)
									i += 2
								else:
									new_word.append(word[i]	)
									i += 1
							snake_case__			:				Dict       =  tuple(__A	)
							snake_case__			:				Dict       =  new_word
							if len(__A	) == 1:
								break
							else:
								snake_case__			:				Tuple       =  get_pairs(__A	)
						snake_case__			:				int       =  " ".join(__A	)
						snake_case__			:				Union[str, Any]       =  word
						return word



					def 	_lowercase					( self					:  Dict							,		__A					:  Optional[Any]	):
						snake_case__			:				str       =  []
						for token in re.findall(self.pat							,		__A	):
							snake_case__			:				List[str]       =  "".join(
							    self.byte_encoder[b] for b in token.encode("utf-8"	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A	).split(" "	)	)
						return bpe_tokens



					def 	_lowercase					( self					:  Optional[int]							,		__A					:  Dict	):
						return self.encoder.get(__A							,		self.encoder.get(self.unk_token	)	)



					def 	_lowercase					( self					:  Optional[int]							,		__A					:  Any	):
						return self.decoder.get(__A	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  List[Any]	):
						snake_case__			:				Any       =  "".join(__A	)
						snake_case__			:				Tuple       =  bytearray([self.byte_decoder[c] for c in text]	).decode("utf-8"							,		errors=self.errors	)
						return text



					def 	_lowercase					( self					:  Any							,		__A					:  str							,		__A					:  Optional[str] = None	):
						if not os.path.isdir(__A	):
							logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
							return
						snake_case__			:				Union[str, Any]       =  os.path.join(
						    __A							,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
						snake_case__			:				int       =  os.path.join(
						    __A							,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]	)

						with open(__A							,		"w"							,		encoding="utf-8"	) as f:
							f.write(json.dumps(self.encoder							,		indent=2							,		sort_keys=__A							,		ensure_ascii=__A	) + "\n"	)

						snake_case__			:				int       =  0
						with open(__A							,		"w"							,		encoding="utf-8"	) as writer:
							writer.write("#version: 0.2\n"	)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,		key=lambda __A	: kv[1]	):
								if index != token_index:
									logger.warning(
									    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
									    " Please check that the tokenizer is not corrupted!"	)
									snake_case__			:				Any       =  token_index
								writer.write(" ".join(__A	) + "\n"	)
								index += 1

						return vocab_file, merge_file



					def 	_lowercase					( self					:  List[str]							,		__A					:  List[int]							,		__A					:  Optional[List[int]] = None	):
						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						snake_case__			:				Any       =  [self.cls_token_id]
						snake_case__			:				Union[str, Any]       =  [self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep



					def 	_lowercase					( self					:  Tuple							,		__A					:  List[int]							,		__A					:  Optional[List[int]] = None							,		__A					:  bool = False	):
						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__A							,		token_ids_a=__A							,		already_has_special_tokens=__A	)

						if token_ids_a is None:
							return [1] + ([0] * len(__A	)) + [1]
						return [1] + ([0] * len(__A	)) + [1, 1] + ([0] * len(__A	)) + [1]



					def 	_lowercase					( self					:  Optional[int]							,		__A					:  List[int]							,		__A					:  Optional[List[int]] = None	):
						snake_case__			:				Optional[int]       =  [self.sep_token_id]
						snake_case__			:				List[str]       =  [self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep	) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]






					def 	_lowercase					( self					:  Dict							,		__A					:  Any							,		__A					:  Optional[int]=False							,		**__A					:  Tuple	):
						snake_case__			:				Tuple       =  kwargs.pop("add_prefix_space"							,		self.add_prefix_space	)
						if (is_split_into_words or add_prefix_space) and (len(__A	) > 0 and not text[0].isspace()):
							snake_case__			:				Optional[Any]       =  " " + text
						return (text, kwargs)











from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
	import PIL


__lowerCamelCase :      int										=				logging.get_logger(__name__)







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							int					,  snake_case_ :							Any       ):
	snake_case__			:				List[str]       =  b.T
	snake_case__			:				Union[str, Any]       =  np.sum(np.square(snake_case_       )					,  axis=1       )
	snake_case__			:				Dict       =  np.sum(np.square(snake_case_       )					,  axis=0       )
	snake_case__			:				Dict       =  np.matmul(snake_case_					,  snake_case_       )
	snake_case__			:				Any       =  aa[:, None] - 2 * ab + ba[None, :]
	return d







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Optional[int]					,  snake_case_ :							Tuple       ):
	snake_case__			:				Tuple       =  x.reshape(-1					,  3       )
	snake_case__			:				int       =  squared_euclidean_distance(snake_case_					,  snake_case_       )
	return np.argmin(snake_case_					,  axis=1       )





class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_						):





					"""simple docstring"""






					a_     =		["pixel_values"]



					def __init__( self					:  str							,		__A					:  Optional[Union[List[List[int]], np.ndarray]] = None							,		__A					:  bool = True							,		__A					:  Dict[str, int] = None							,		__A					:  PILImageResampling = PILImageResampling.BILINEAR							,		__A					:  bool = True							,		__A					:  bool = True							,		**__A					:  Union[str, Any]							,		):
						super().__init__(**__A	)
						snake_case__			:				Optional[int]       =  size if size is not None else {"height": 2_5_6, "width": 2_5_6}
						snake_case__			:				List[Any]       =  get_size_dict(__A	)
						snake_case__			:				Any       =  np.array(__A	) if clusters is not None else None
						snake_case__			:				Optional[Any]       =  do_resize
						snake_case__			:				Any       =  size
						snake_case__			:				List[Any]       =  resample
						snake_case__			:				List[Any]       =  do_normalize
						snake_case__			:				Dict       =  do_color_quantize



					def 	_lowercase					( self					:  List[Any]							,		__A					:  np.ndarray							,		__A					:  Dict[str, int]							,		__A					:  PILImageResampling = PILImageResampling.BILINEAR							,		__A					:  Optional[Union[str, ChannelDimension]] = None							,		**__A					:  int							,		):
						snake_case__			:				List[Any]       =  get_size_dict(__A	)
						if "height" not in size or "width" not in size:
							raise ValueError(f'''Size dictionary must contain both height and width keys. Got {size.keys()}'''	)
						return resize(
						    __A							,		size=(size["height"], size["width"])							,		resample=__A							,		data_format=__A							,		**__A	)



					def 	_lowercase					( self					:  List[Any]							,		__A					:  np.ndarray							,		__A					:  Optional[Union[str, ChannelDimension]] = None							,		):
						snake_case__			:				List[str]       =  rescale(image=__A							,		scale=1 / 1_2_7.5							,		data_format=__A	)
						snake_case__			:				List[Any]       =  image - 1
						return image






					def 	_lowercase					( self					:  Dict							,		__A					:  ImageInput							,		__A					:  bool = None							,		__A					:  Dict[str, int] = None							,		__A					:  PILImageResampling = None							,		__A					:  bool = None							,		__A					:  Optional[bool] = None							,		__A					:  Optional[Union[List[List[int]], np.ndarray]] = None							,		__A					:  Optional[Union[str, TensorType]] = None							,		__A					:  Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST							,		**__A					:  Optional[int]							,		):
						snake_case__			:				Any       =  do_resize if do_resize is not None else self.do_resize
						snake_case__			:				Union[str, Any]       =  size if size is not None else self.size
						snake_case__			:				Union[str, Any]       =  get_size_dict(__A	)
						snake_case__			:				Optional[Any]       =  resample if resample is not None else self.resample
						snake_case__			:				Tuple       =  do_normalize if do_normalize is not None else self.do_normalize
						snake_case__			:				Optional[Any]       =  do_color_quantize if do_color_quantize is not None else self.do_color_quantize
						snake_case__			:				Union[str, Any]       =  clusters if clusters is not None else self.clusters
						snake_case__			:				Union[str, Any]       =  np.array(__A	)

						snake_case__			:				Any       =  make_list_of_images(__A	)

						if not valid_images(__A	):
							raise ValueError(
							    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
							    "torch.Tensor, tf.Tensor or jax.ndarray."	)

						if do_resize and size is None or resample is None:
							raise ValueError("Size and resample must be specified if do_resize is True."	)

						if do_color_quantize and clusters is None:
							raise ValueError("Clusters must be specified if do_color_quantize is True."	)

						# All transformations expect numpy arrays.
						snake_case__			:				Optional[Any]       =  [to_numpy_array(__A	) for image in images]

						if do_resize:
							snake_case__			:				List[str]       =  [self.resize(image=__A							,		size=__A							,		resample=__A	) for image in images]

						if do_normalize:
							snake_case__			:				Union[str, Any]       =  [self.normalize(image=__A	) for image in images]

						if do_color_quantize:
							snake_case__			:				int       =  [to_channel_dimension_format(__A							,		ChannelDimension.LAST	) for image in images]
							# color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
							snake_case__			:				int       =  np.array(__A	)
							snake_case__			:				Dict       =  color_quantize(__A							,		__A	).reshape(images.shape[:-1]	)

							# flatten to (batch_size, height*width)
							snake_case__			:				str       =  images.shape[0]
							snake_case__			:				str       =  images.reshape(__A							,		-1	)

							# We need to convert back to a list of images to keep consistent behaviour across processors.
							snake_case__			:				Union[str, Any]       =  list(__A	)
						else:
							snake_case__			:				Any       =  [to_channel_dimension_format(__A							,		__A	) for image in images]

						snake_case__			:				Optional[int]       =  {"input_ids": images}
						return BatchFeature(data=__A							,		tensor_type=__A	)



