


"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__       :							int        =  logging.get_logger(__name__)

a__       :							int        =  '''▁'''

a__       :							str        =  {'''vocab_file''': '''sentencepiece.bpe.model'''}

a__       :							int        =  {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

a__       :							int        =  {
    '''xlm-roberta-base''': 5_1_2,
    '''xlm-roberta-large''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_1_2,
}




class 				UpperCamelCase_			(				UpperCamelCase):

	"""simple docstring"""





	snake_case__       :						Any					      =    VOCAB_FILES_NAMES
	snake_case__       :						List[Any]					      =    PRETRAINED_VOCAB_FILES_MAP
	snake_case__       :						List[str]					      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__       :						Tuple					      =    ["input_ids", "attention_mask"]




	def __init__(				self    :       Optional[Any]    ,    UpperCAmelCase__    :       Any    ,    UpperCAmelCase__    :       Optional[Any]="<s>"    ,    UpperCAmelCase__    :       List[str]="</s>"    ,    UpperCAmelCase__    :       List[Any]="</s>"    ,    UpperCAmelCase__    :       Tuple="<s>"    ,    UpperCAmelCase__    :       List[str]="<unk>"    ,    UpperCAmelCase__    :       List[Any]="<pad>"    ,    UpperCAmelCase__    :       int="<mask>"    ,    UpperCAmelCase__    :       Optional[Dict[str, Any]] = None    ,    **UpperCAmelCase__    :       Union[str, Any]    ,    ) ->			None:
						# Mask token behave like a normal word, i.e. include the space before it
						__SCREAMING_SNAKE_CASE		     =       AddedToken(UpperCAmelCase__    ,    lstrip=UpperCAmelCase__    ,    rstrip=UpperCAmelCase__     ) if isinstance(UpperCAmelCase__    ,    UpperCAmelCase__     ) else mask_token

						__SCREAMING_SNAKE_CASE		     =       {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    bos_token=UpperCAmelCase__    ,    eos_token=UpperCAmelCase__    ,    unk_token=UpperCAmelCase__    ,    sep_token=UpperCAmelCase__    ,    cls_token=UpperCAmelCase__    ,    pad_token=UpperCAmelCase__    ,    mask_token=UpperCAmelCase__    ,    sp_model_kwargs=self.sp_model_kwargs    ,    **UpperCAmelCase__    ,    )

						__SCREAMING_SNAKE_CASE		     =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(str(UpperCAmelCase__     )     )
						__SCREAMING_SNAKE_CASE		     =       vocab_file

						# Original fairseq vocab and spm vocab must be "aligned":
						# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
						# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
						# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
						# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

						# Mimic fairseq token-to-id alignment for the first 4 token
						__SCREAMING_SNAKE_CASE		     =       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

						# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
						__SCREAMING_SNAKE_CASE		     =       1

						__SCREAMING_SNAKE_CASE		     =       len(self.sp_model     ) + self.fairseq_offset
						__SCREAMING_SNAKE_CASE		     =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




	def __getstate__(				self    :       Optional[Any]     ) ->			Union[str, Any]:
						__SCREAMING_SNAKE_CASE		     =       self.__dict__.copy()
						__SCREAMING_SNAKE_CASE		     =       None
						__SCREAMING_SNAKE_CASE		     =       self.sp_model.serialized_model_proto()
						return state




	def __setstate__(				self    :       Optional[Any]    ,    UpperCAmelCase__    :       Optional[Any]     ) ->			str:
						__SCREAMING_SNAKE_CASE		     =       d

						# for backward compatibility
						if not hasattr(self    ,    "sp_model_kwargs"     ):
											__SCREAMING_SNAKE_CASE		     =       {}

						__SCREAMING_SNAKE_CASE		     =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.LoadFromSerializedProto(self.sp_model_proto     )




	def 						UpperCAmelCase_   (				self    :       List[Any]    ,    UpperCAmelCase__    :       List[int]    ,    UpperCAmelCase__    :       Optional[List[int]] = None     ) ->			List[int]:

						if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__SCREAMING_SNAKE_CASE		     =       [self.cls_token_id]
						__SCREAMING_SNAKE_CASE		     =       [self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep




	def 						UpperCAmelCase_   (				self    :       Dict    ,    UpperCAmelCase__    :       List[int]    ,    UpperCAmelCase__    :       Optional[List[int]] = None    ,    UpperCAmelCase__    :       bool = False     ) ->			List[int]:

						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=UpperCAmelCase__    ,    token_ids_a=UpperCAmelCase__    ,    already_has_special_tokens=UpperCAmelCase__     )

						if token_ids_a is None:
											return [1] + ([0] * len(UpperCAmelCase__     )) + [1]
						return [1] + ([0] * len(UpperCAmelCase__     )) + [1, 1] + ([0] * len(UpperCAmelCase__     )) + [1]




	def 						UpperCAmelCase_   (				self    :       List[str]    ,    UpperCAmelCase__    :       List[int]    ,    UpperCAmelCase__    :       Optional[List[int]] = None     ) ->			List[int]:
						__SCREAMING_SNAKE_CASE		     =       [self.sep_token_id]
						__SCREAMING_SNAKE_CASE		     =       [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep     ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]




	@property
	def 						UpperCAmelCase_   (				self    :       Union[str, Any]     ) ->			Optional[Any]:
						return len(self.sp_model     ) + self.fairseq_offset + 1  # Add the <mask> token




	def 						UpperCAmelCase_   (				self    :       Tuple     ) ->			List[str]:
						__SCREAMING_SNAKE_CASE		     =       {self.convert_ids_to_tokens(UpperCAmelCase__     ): i for i in range(self.vocab_size     )}
						vocab.update(self.added_tokens_encoder     )
						return vocab




	def 						UpperCAmelCase_   (				self    :       List[Any]    ,    UpperCAmelCase__    :       str     ) ->			List[str]:
						return self.sp_model.encode(UpperCAmelCase__    ,    out_type=UpperCAmelCase__     )




	def 						UpperCAmelCase_   (				self    :       str    ,    UpperCAmelCase__    :       Any     ) ->			Dict:
						if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
						__SCREAMING_SNAKE_CASE		     =       self.sp_model.PieceToId(UpperCAmelCase__     )

						# Need to return unknown token if the SP model returned 0
						return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




	def 						UpperCAmelCase_   (				self    :       List[str]    ,    UpperCAmelCase__    :       Optional[Any]     ) ->			Optional[int]:
						if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
						return self.sp_model.IdToPiece(index - self.fairseq_offset     )




	def 						UpperCAmelCase_   (				self    :       List[str]    ,    UpperCAmelCase__    :       Any     ) ->			Any:
						__SCREAMING_SNAKE_CASE		     =       "".join(UpperCAmelCase__     ).replace(UpperCAmelCase__    ,    " "     ).strip()
						return out_string




	def 						UpperCAmelCase_   (				self    :       Any    ,    UpperCAmelCase__    :       str    ,    UpperCAmelCase__    :       Optional[str] = None     ) ->			Tuple[str]:
						if not os.path.isdir(UpperCAmelCase__     ):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
											return
						__SCREAMING_SNAKE_CASE		     =       os.path.join(
						    UpperCAmelCase__    ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

						if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCAmelCase__     ) and os.path.isfile(self.vocab_file     ):
											copyfile(self.vocab_file    ,    UpperCAmelCase__     )
						elif not os.path.isfile(self.vocab_file     ):
											with open(UpperCAmelCase__    ,    "wb"     ) as fi:
																__SCREAMING_SNAKE_CASE		     =       self.sp_model.serialized_model_proto()
																fi.write(UpperCAmelCase__     )

						return (out_vocab_file,)




"""simple docstring"""







import math
from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    is_torch_available,
    is_torch_tensor,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_torch_available():
		import torch

if is_vision_available():
		import PIL


a__       :							Dict        =  logging.get_logger(__name__)




def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					def constraint_to_multiple_of(lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_=0						,		lowerCAmelCase_=None     ):
										__SCREAMING_SNAKE_CASE		     =       round(val / multiple     ) * multiple

										if max_val is not None and x > max_val:
															__SCREAMING_SNAKE_CASE		     =       math.floor(val / multiple     ) * multiple

										if x < min_val:
															__SCREAMING_SNAKE_CASE		     =       math.ceil(val / multiple     ) * multiple

										return x

					__SCREAMING_SNAKE_CASE		     =       (output_size, output_size) if isinstance(lowerCAmelCase_						,		lowerCAmelCase_     ) else output_size

					__SCREAMING_SNAKE_CASE     ,						__SCREAMING_SNAKE_CASE		     =       get_image_size(lowerCAmelCase_     )
					__SCREAMING_SNAKE_CASE     ,						__SCREAMING_SNAKE_CASE		     =       output_size

					# determine new height and width
					__SCREAMING_SNAKE_CASE		     =       output_height / input_height
					__SCREAMING_SNAKE_CASE		     =       output_width / input_width

					if keep_aspect_ratio:
										# scale as little as possible
										if abs(1 - scale_width     ) < abs(1 - scale_height     ):
															# fit width
															__SCREAMING_SNAKE_CASE		     =       scale_width
										else:
															# fit height
															__SCREAMING_SNAKE_CASE		     =       scale_height

					__SCREAMING_SNAKE_CASE		     =       constraint_to_multiple_of(scale_height * input_height						,		multiple=lowerCAmelCase_     )
					__SCREAMING_SNAKE_CASE		     =       constraint_to_multiple_of(scale_width * input_width						,		multiple=lowerCAmelCase_     )

					return (new_height, new_width)




class 				UpperCamelCase_			(				UpperCamelCase):

	"""simple docstring"""





	snake_case__       :						List[Any]					      =    ["pixel_values"]




	def __init__(				self    :       int    ,    UpperCAmelCase__    :       bool = True    ,    UpperCAmelCase__    :       Dict[str, int] = None    ,    UpperCAmelCase__    :       PILImageResampling = PILImageResampling.BILINEAR    ,    UpperCAmelCase__    :       bool = False    ,    UpperCAmelCase__    :       int = 1    ,    UpperCAmelCase__    :       bool = True    ,    UpperCAmelCase__    :       Union[int, float] = 1 / 2_5_5    ,    UpperCAmelCase__    :       bool = True    ,    UpperCAmelCase__    :       Optional[Union[float, List[float]]] = None    ,    UpperCAmelCase__    :       Optional[Union[float, List[float]]] = None    ,    **UpperCAmelCase__    :       Optional[Any]    ,    ) ->			None:
						super().__init__(**UpperCAmelCase__     )
						__SCREAMING_SNAKE_CASE		     =       size if size is not None else {"height": 3_8_4, "width": 3_8_4}
						__SCREAMING_SNAKE_CASE		     =       get_size_dict(UpperCAmelCase__     )
						__SCREAMING_SNAKE_CASE		     =       do_resize
						__SCREAMING_SNAKE_CASE		     =       size
						__SCREAMING_SNAKE_CASE		     =       keep_aspect_ratio
						__SCREAMING_SNAKE_CASE		     =       ensure_multiple_of
						__SCREAMING_SNAKE_CASE		     =       resample
						__SCREAMING_SNAKE_CASE		     =       do_rescale
						__SCREAMING_SNAKE_CASE		     =       rescale_factor
						__SCREAMING_SNAKE_CASE		     =       do_normalize
						__SCREAMING_SNAKE_CASE		     =       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
						__SCREAMING_SNAKE_CASE		     =       image_std if image_std is not None else IMAGENET_STANDARD_STD




	def 						UpperCAmelCase_   (				self    :       Any    ,    UpperCAmelCase__    :       np.ndarray    ,    UpperCAmelCase__    :       Dict[str, int]    ,    UpperCAmelCase__    :       bool = False    ,    UpperCAmelCase__    :       int = 1    ,    UpperCAmelCase__    :       PILImageResampling = PILImageResampling.BICUBIC    ,    UpperCAmelCase__    :       Optional[Union[str, ChannelDimension]] = None    ,    **UpperCAmelCase__    :       Dict    ,    ) ->			np.ndarray:
						__SCREAMING_SNAKE_CASE		     =       get_size_dict(UpperCAmelCase__     )
						if "height" not in size or "width" not in size:
											raise ValueError(F"""The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"""     )
						__SCREAMING_SNAKE_CASE		     =       get_resize_output_image_size(
						    UpperCAmelCase__    ,    output_size=(size["height"], size["width"])    ,    keep_aspect_ratio=UpperCAmelCase__    ,    multiple=UpperCAmelCase__    ,    )
						return resize(UpperCAmelCase__    ,    size=UpperCAmelCase__    ,    resample=UpperCAmelCase__    ,    data_format=UpperCAmelCase__    ,    **UpperCAmelCase__     )




	def 						UpperCAmelCase_   (				self    :       Tuple    ,    UpperCAmelCase__    :       np.ndarray    ,    UpperCAmelCase__    :       Union[int, float]    ,    UpperCAmelCase__    :       Optional[Union[str, ChannelDimension]] = None    ,    **UpperCAmelCase__    :       Union[str, Any]    ,    ) ->			Any:
						return rescale(UpperCAmelCase__    ,    scale=UpperCAmelCase__    ,    data_format=UpperCAmelCase__    ,    **UpperCAmelCase__     )




	def 						UpperCAmelCase_   (				self    :       str    ,    UpperCAmelCase__    :       np.ndarray    ,    UpperCAmelCase__    :       Union[float, List[float]]    ,    UpperCAmelCase__    :       Union[float, List[float]]    ,    UpperCAmelCase__    :       Optional[Union[str, ChannelDimension]] = None    ,    **UpperCAmelCase__    :       Optional[Any]    ,    ) ->			np.ndarray:
						return normalize(UpperCAmelCase__    ,    mean=UpperCAmelCase__    ,    std=UpperCAmelCase__    ,    data_format=UpperCAmelCase__    ,    **UpperCAmelCase__     )




	def 						UpperCAmelCase_   (				self    :       List[Any]    ,    UpperCAmelCase__    :       ImageInput    ,    UpperCAmelCase__    :       bool = None    ,    UpperCAmelCase__    :       int = None    ,    UpperCAmelCase__    :       bool = None    ,    UpperCAmelCase__    :       int = None    ,    UpperCAmelCase__    :       PILImageResampling = None    ,    UpperCAmelCase__    :       bool = None    ,    UpperCAmelCase__    :       float = None    ,    UpperCAmelCase__    :       bool = None    ,    UpperCAmelCase__    :       Optional[Union[float, List[float]]] = None    ,    UpperCAmelCase__    :       Optional[Union[float, List[float]]] = None    ,    UpperCAmelCase__    :       Optional[Union[str, TensorType]] = None    ,    UpperCAmelCase__    :       ChannelDimension = ChannelDimension.FIRST    ,    **UpperCAmelCase__    :       int    ,    ) ->			PIL.Image.Image:
						__SCREAMING_SNAKE_CASE		     =       do_resize if do_resize is not None else self.do_resize
						__SCREAMING_SNAKE_CASE		     =       size if size is not None else self.size
						__SCREAMING_SNAKE_CASE		     =       get_size_dict(UpperCAmelCase__     )
						__SCREAMING_SNAKE_CASE		     =       keep_aspect_ratio if keep_aspect_ratio is not None else self.keep_aspect_ratio
						__SCREAMING_SNAKE_CASE		     =       ensure_multiple_of if ensure_multiple_of is not None else self.ensure_multiple_of
						__SCREAMING_SNAKE_CASE		     =       resample if resample is not None else self.resample
						__SCREAMING_SNAKE_CASE		     =       do_rescale if do_rescale is not None else self.do_rescale
						__SCREAMING_SNAKE_CASE		     =       rescale_factor if rescale_factor is not None else self.rescale_factor
						__SCREAMING_SNAKE_CASE		     =       do_normalize if do_normalize is not None else self.do_normalize
						__SCREAMING_SNAKE_CASE		     =       image_mean if image_mean is not None else self.image_mean
						__SCREAMING_SNAKE_CASE		     =       image_std if image_std is not None else self.image_std

						__SCREAMING_SNAKE_CASE		     =       make_list_of_images(UpperCAmelCase__     )

						if not valid_images(UpperCAmelCase__     ):
											raise ValueError(
											    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
											    "torch.Tensor, tf.Tensor or jax.ndarray."     )

						if do_resize and size is None or resample is None:
											raise ValueError("Size and resample must be specified if do_resize is True."     )

						if do_rescale and rescale_factor is None:
											raise ValueError("Rescale factor must be specified if do_rescale is True."     )

						if do_normalize and (image_mean is None or image_std is None):
											raise ValueError("Image mean and std must be specified if do_normalize is True."     )

						# All transformations expect numpy arrays.
						__SCREAMING_SNAKE_CASE		     =       [to_numpy_array(UpperCAmelCase__     ) for image in images]

						if do_resize:
											__SCREAMING_SNAKE_CASE		     =       [self.resize(image=UpperCAmelCase__    ,    size=UpperCAmelCase__    ,    resample=UpperCAmelCase__     ) for image in images]

						if do_rescale:
											__SCREAMING_SNAKE_CASE		     =       [self.rescale(image=UpperCAmelCase__    ,    scale=UpperCAmelCase__     ) for image in images]

						if do_normalize:
											__SCREAMING_SNAKE_CASE		     =       [self.normalize(image=UpperCAmelCase__    ,    mean=UpperCAmelCase__    ,    std=UpperCAmelCase__     ) for image in images]

						__SCREAMING_SNAKE_CASE		     =       [to_channel_dimension_format(UpperCAmelCase__    ,    UpperCAmelCase__     ) for image in images]

						__SCREAMING_SNAKE_CASE		     =       {"pixel_values": images}
						return BatchFeature(data=UpperCAmelCase__    ,    tensor_type=UpperCAmelCase__     )




	def 						UpperCAmelCase_   (				self    :       Optional[int]    ,    UpperCAmelCase__    :       List[str]    ,    UpperCAmelCase__    :       List[Tuple] = None     ) ->			Optional[Any]:
						__SCREAMING_SNAKE_CASE		     =       outputs.logits

						# Resize logits and compute semantic segmentation maps
						if target_sizes is not None:
											if len(UpperCAmelCase__     ) != len(UpperCAmelCase__     ):
																raise ValueError(
																    "Make sure that you pass in as many target sizes as the batch dimension of the logits"     )

											if is_torch_tensor(UpperCAmelCase__     ):
																__SCREAMING_SNAKE_CASE		     =       target_sizes.numpy()

											__SCREAMING_SNAKE_CASE		     =       []

											for idx in range(len(UpperCAmelCase__     )     ):
																__SCREAMING_SNAKE_CASE		     =       torch.nn.functional.interpolate(
																    logits[idx].unsqueeze(dim=0     )    ,    size=target_sizes[idx]    ,    mode="bilinear"    ,    align_corners=UpperCAmelCase__     )
																__SCREAMING_SNAKE_CASE		     =       resized_logits[0].argmax(dim=0     )
																semantic_segmentation.append(UpperCAmelCase__     )
						else:
											__SCREAMING_SNAKE_CASE		     =       logits.argmax(dim=1     )
											__SCREAMING_SNAKE_CASE		     =       [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]     )]

						return semantic_segmentation
