


'''simple docstring'''






import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
		import PIL.Image

		from .features import FeatureType


__UpperCAmelCase					       =							None
__UpperCAmelCase					       =							'''<''' if sys.byteorder == '''little''' else '''>'''
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
__UpperCAmelCase					       =							[
    np.dtype('''|b1'''),
    np.dtype('''|u1'''),
    np.dtype('''<u2'''),
    np.dtype('''>u2'''),
    np.dtype('''<i2'''),
    np.dtype('''>i2'''),
    np.dtype('''<u4'''),
    np.dtype('''>u4'''),
    np.dtype('''<i4'''),
    np.dtype('''>i4'''),
    np.dtype('''<f4'''),
    np.dtype('''>f4'''),
    np.dtype('''<f8'''),
    np.dtype('''>f8'''),
]




@dataclass
class 	a__       :





							'''simple docstring'''






							lowercase__    :     bool					=   True
							lowercase__    :     Optional[str]					=   None
							# Automatically constructed
							lowercase__    :     ClassVar[str]					=   "PIL.Image.Image"
							lowercase__    :     ClassVar[Any]					=   pa.struct({"bytes": pa.binary(), "path": pa.string()}						)
							lowercase__    :     str					=   field(default="Image"       ,       init=a__       ,       repr=a__						)







							def __call__(	self     ) ->							Union[str, Any]:
														return self.pa_type







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							dict:
														if config.PIL_AVAILABLE:
																					import PIL.Image
														else:
																					raise ImportError('''To support encoding images, please install \'Pillow\'.'''     )

														if isinstance(lowerCamelCase_      ,					lowerCamelCase_     ):
																					lowerCAmelCase__      =			np.array(lowerCamelCase_     )

														if isinstance(lowerCamelCase_      ,					lowerCamelCase_     ):
																					return {"path": value, "bytes": None}
														elif isinstance(lowerCamelCase_      ,					lowerCamelCase_     ):
																					return {"path": None, "bytes": value}
														elif isinstance(lowerCamelCase_      ,					np.ndarray     ):
																					# convert the image array to PNG/TIFF bytes
																					return encode_np_array(lowerCamelCase_     )
														elif isinstance(lowerCamelCase_      ,					PIL.Image.Image     ):
																					# convert the PIL image to bytes (default format is PNG/TIFF)
																					return encode_pil_image(lowerCamelCase_     )
														elif value.get('''path'''     ) is not None and os.path.isfile(value['''path''']     ):
																					# we set "bytes": None to not duplicate the data if they're already available locally
																					return {"bytes": None, "path": value.get('''path'''     )}
														elif value.get('''bytes'''     ) is not None or value.get('''path'''     ) is not None:
																					# store the image bytes, and path is used to infer the image format using the file extension
																					return {"bytes": value.get('''bytes'''     ), "path": value.get('''path'''     )}
														else:
																					raise ValueError(
																					    F"""An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."""     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_=None     ) ->							"PIL.Image.Image":
														if not self.decode:
																					raise RuntimeError('''Decoding is disabled for this feature. Please use Image(decode=True) instead.'''     )

														if config.PIL_AVAILABLE:
																					import PIL.Image
														else:
																					raise ImportError('''To support decoding images, please install \'Pillow\'.'''     )

														if token_per_repo_id is None:
																					lowerCAmelCase__      =			{}

														lowerCAmelCase__     ,      lowerCAmelCase__      =			value['''path'''], value['''bytes''']
														if bytes_ is None:
																					if path is None:
																												raise ValueError(F"""An image should have one of 'path' or 'bytes' but both are None in {value}."""     )
																					else:
																												if is_local_path(lowerCamelCase_     ):
																																			lowerCAmelCase__      =			PIL.Image.open(lowerCamelCase_     )
																												else:
																																			lowerCAmelCase__      =			path.split('''::'''     )[-1]
																																			try:
																																										lowerCAmelCase__      =			string_to_dict(lowerCamelCase_      ,					config.HUB_DATASETS_URL     )['''repo_id''']
																																										lowerCAmelCase__      =			token_per_repo_id.get(lowerCamelCase_     )
																																			except ValueError:
																																										lowerCAmelCase__      =			None
																																			with xopen(lowerCamelCase_      ,					'''rb'''      ,					use_auth_token=lowerCamelCase_     ) as f:
																																										lowerCAmelCase__      =			BytesIO(f.read()     )
																																			lowerCAmelCase__      =			PIL.Image.open(bytes_     )
														else:
																					lowerCAmelCase__      =			PIL.Image.open(BytesIO(bytes_     )     )
														image.load()  # to avoid "Too many open files" errors
														return image







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Union["FeatureType", Dict[str, "FeatureType"]]:
														from .features import Value

														return (
														    self
														    if self.decode
														    else {
														        "bytes": Value('''binary'''     ),
														        "path": Value('''string'''     ),
														    }
														)







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							pa.StructArray:
														if pa.types.is_string(storage.type     ):
																					lowerCAmelCase__      =			pa.array([None] * len(lowerCamelCase_     )      ,					type=pa.binary()     )
																					lowerCAmelCase__      =			pa.StructArray.from_arrays([bytes_array, storage]      ,					['''bytes''', '''path''']      ,					mask=storage.is_null()     )
														elif pa.types.is_binary(storage.type     ):
																					lowerCAmelCase__      =			pa.array([None] * len(lowerCamelCase_     )      ,					type=pa.string()     )
																					lowerCAmelCase__      =			pa.StructArray.from_arrays([storage, path_array]      ,					['''bytes''', '''path''']      ,					mask=storage.is_null()     )
														elif pa.types.is_struct(storage.type     ):
																					if storage.type.get_field_index('''bytes'''     ) >= 0:
																												lowerCAmelCase__      =			storage.field('''bytes'''     )
																					else:
																												lowerCAmelCase__      =			pa.array([None] * len(lowerCamelCase_     )      ,					type=pa.binary()     )
																					if storage.type.get_field_index('''path'''     ) >= 0:
																												lowerCAmelCase__      =			storage.field('''path'''     )
																					else:
																												lowerCAmelCase__      =			pa.array([None] * len(lowerCamelCase_     )      ,					type=pa.string()     )
																					lowerCAmelCase__      =			pa.StructArray.from_arrays([bytes_array, path_array]      ,					['''bytes''', '''path''']      ,					mask=storage.is_null()     )
														elif pa.types.is_list(storage.type     ):
																					lowerCAmelCase__      =			pa.array(
																					    [encode_np_array(np.array(lowerCamelCase_     )     )['''bytes'''] if arr is not None else None for arr in storage.to_pylist()]      ,					type=pa.binary()      ,					)
																					lowerCAmelCase__      =			pa.array([None] * len(lowerCamelCase_     )      ,					type=pa.string()     )
																					lowerCAmelCase__      =			pa.StructArray.from_arrays(
																					    [bytes_array, path_array]      ,					['''bytes''', '''path''']      ,					mask=bytes_array.is_null()     )
														return array_cast(lowerCamelCase_      ,					self.pa_type     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							pa.StructArray:

														@no_op_if_value_is_null
														def path_to_bytes(lowerCamelCase_     ):
																					with xopen(lowerCamelCase_      ,					'''rb'''     ) as f:
																												lowerCAmelCase__      =			f.read()
																					return bytes_

														lowerCAmelCase__      =			pa.array(
														    [
														        (path_to_bytes(x['''path''']     ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
														        for x in storage.to_pylist()
														    ]      ,					type=pa.binary()      ,					)
														lowerCAmelCase__      =			pa.array(
														    [os.path.basename(lowerCamelCase_     ) if path is not None else None for path in storage.field('''path'''     ).to_pylist()]      ,					type=pa.string()      ,					)
														lowerCAmelCase__      =			pa.StructArray.from_arrays([bytes_array, path_array]      ,					['''bytes''', '''path''']      ,					mask=bytes_array.is_null()     )
														return array_cast(lowerCamelCase_      ,					self.pa_type     )


def       _snake_case     (  )		->					List[str]:
							if config.PIL_AVAILABLE:
														import PIL.Image
							else:
														raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

							global _IMAGE_COMPRESSION_FORMATS
							if _IMAGE_COMPRESSION_FORMATS is None:
														PIL.Image.init()
														lowerCAmelCase__      =			list(set(PIL.Image.OPEN.keys() ) & set(PIL.Image.SAVE.keys() ) )
							return _IMAGE_COMPRESSION_FORMATS


def       _snake_case     (  A )		->					bytes:
							lowerCAmelCase__      =			BytesIO()
							if image.format in list_image_compression_formats():
														lowerCAmelCase__      =			image.format
							else:
														lowerCAmelCase__      =			'''PNG''' if image.mode in ['''1''', '''L''', '''LA''', '''RGB''', '''RGBA'''] else '''TIFF'''
							image.save(A				,					format=A )
							return buffer.getvalue()


def       _snake_case     (  A )		->					dict:
							if hasattr(A				,					'''filename''' ) and image.filename != "":
														return {"path": image.filename, "bytes": None}
							else:
														return {"path": None, "bytes": image_to_bytes(A )}


def       _snake_case     (  A )		->					dict:
							if config.PIL_AVAILABLE:
														import PIL.Image
							else:
														raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

							lowerCAmelCase__      =			array.dtype
							lowerCAmelCase__      =			dtype.byteorder if dtype.byteorder != '''=''' else _NATIVE_BYTEORDER
							lowerCAmelCase__      =			dtype.kind
							lowerCAmelCase__      =			dtype.itemsize

							lowerCAmelCase__      =			None

							# Multi-channel array case (only np.dtype("|u1") is allowed)
							if array.shape[2:]:
														lowerCAmelCase__      =			np.dtype('''|u1''' )
														if dtype_kind not in ["u", "i"]:
																					raise TypeError(
																					    F"""Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.""" )
														if dtype is not dest_dtype:
																					warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'""" )
    # Exact match
							elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
														lowerCAmelCase__      =			dtype
							else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
														while dtype_itemsize >= 1:
																					lowerCAmelCase__      =			dtype_byteorder + dtype_kind + str(A )
																					lowerCAmelCase__      =			np.dtype(A )
																					if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
																												warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'""" )
																												break
																					else:
																												dtype_itemsize //= 2

							if dest_dtype is None:
														raise TypeError(
														    F"""Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}""" )
							lowerCAmelCase__      =			PIL.Image.fromarray(array.astype(A ) )
							return {"path": None, "bytes": image_to_bytes(A )}






def       _snake_case     (  A )		->					List[dict]:
							if config.PIL_AVAILABLE:
														import PIL.Image
							else:
														raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

							if objs:
														lowerCAmelCase__     ,      lowerCAmelCase__      =			first_non_null_value(A )
														if isinstance(A				,					A ):
																					return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
														if isinstance(A				,					np.ndarray ):
																					lowerCAmelCase__      =			no_op_if_value_is_null(A )
																					return [obj_to_image_dict_func(A ) for obj in objs]
														elif isinstance(A				,					PIL.Image.Image ):
																					lowerCAmelCase__      =			no_op_if_value_is_null(A )
																					return [obj_to_image_dict_func(A ) for obj in objs]
														else:
																					return objs
							else:
														return objs



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCAmelCase					       =							{
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCAmelCase					       =							{
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     Optional[Any]					=   VOCAB_FILES_NAMES
							lowercase__    :     int					=   PRETRAINED_VOCAB_FILES_MAP
							lowercase__    :     Any					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__    :     List[Any]					=   ["input_ids", "attention_mask"]
							lowercase__    :     Dict					=   RobertaTokenizer







							def __init__(	self      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_="replace"      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="<unk>"      ,					lowerCamelCase_="<pad>"      ,					lowerCamelCase_="<mask>"      ,					lowerCamelCase_=False      ,					lowerCamelCase_=True      ,					**lowerCamelCase_      ,					) ->							Tuple:
														super().__init__(
														    lowerCamelCase_      ,					lowerCamelCase_      ,					tokenizer_file=lowerCamelCase_      ,					errors=lowerCamelCase_      ,					bos_token=lowerCamelCase_      ,					eos_token=lowerCamelCase_      ,					sep_token=lowerCamelCase_      ,					cls_token=lowerCamelCase_      ,					unk_token=lowerCamelCase_      ,					pad_token=lowerCamelCase_      ,					mask_token=lowerCamelCase_      ,					add_prefix_space=lowerCamelCase_      ,					trim_offsets=lowerCamelCase_      ,					**lowerCamelCase_      ,					)

														lowerCAmelCase__      =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
														if pre_tok_state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																					lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					pre_tok_state.pop('''type'''     )     )
																					lowerCAmelCase__      =			add_prefix_space
																					lowerCAmelCase__      =			pre_tok_class(**lowerCamelCase_     )

														lowerCAmelCase__      =			add_prefix_space

														lowerCAmelCase__      =			'''post_processor'''
														lowerCAmelCase__      =			getattr(self.backend_tokenizer      ,					lowerCamelCase_      ,					lowerCamelCase_     )
														if tokenizer_component_instance:
																					lowerCAmelCase__      =			json.loads(tokenizer_component_instance.__getstate__()     )

																					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																					if "sep" in state:
																												lowerCAmelCase__      =			tuple(state['''sep''']     )
																					if "cls" in state:
																												lowerCAmelCase__      =			tuple(state['''cls''']     )

																					lowerCAmelCase__      =			False

																					if state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																												lowerCAmelCase__      =			add_prefix_space
																												lowerCAmelCase__      =			True

																					if state.get('''trim_offsets'''      ,					lowerCamelCase_     ) != trim_offsets:
																												lowerCAmelCase__      =			trim_offsets
																												lowerCAmelCase__      =			True

																					if changes_to_apply:
																												lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					state.pop('''type'''     )     )
																												lowerCAmelCase__      =			component_class(**lowerCamelCase_     )
																												setattr(self.backend_tokenizer      ,					lowerCamelCase_      ,					lowerCamelCase_     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														if self._mask_token is None:
																					if self.verbose:
																												logger.error('''Using mask_token, but it is not set yet.'''     )
																					return None
														return str(self._mask_token     )







							@mask_token.setter
							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Optional[Any]:
														lowerCAmelCase__      =			AddedToken(lowerCamelCase_      ,					lstrip=lowerCamelCase_      ,					rstrip=lowerCamelCase_     ) if isinstance(lowerCamelCase_      ,					lowerCamelCase_     ) else value
														lowerCAmelCase__      =			value







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )
														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._batch_encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )

														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							Tuple[str]:
														lowerCAmelCase__      =			self._tokenizer.model.save(lowerCamelCase_      ,					name=lowerCamelCase_     )
														return tuple(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_=None     ) ->							Tuple:
														lowerCAmelCase__      =			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
														if token_ids_a is None:
																					return output

														return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							List[int]:
														lowerCAmelCase__      =			[self.sep_token_id]
														lowerCAmelCase__      =			[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep     ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]