




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


__a							     =					{}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a							     =					['NllbTokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a							     =					['NllbTokenizerFast']


if TYPE_CHECKING:
     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb import NllbTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb_fast import NllbTokenizerFast

else:
     import sys

     __a							     =					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
     import PIL.Image

     from .features import FeatureType


__a							     =					None
__a							     =					'<' if sys.byteorder == 'little' else '>'
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
__a							     =					[
    np.dtype('|b1'),
    np.dtype('|u1'),
    np.dtype('<u2'),
    np.dtype('>u2'),
    np.dtype('<i2'),
    np.dtype('>i2'),
    np.dtype('<u4'),
    np.dtype('>u4'),
    np.dtype('<i4'),
    np.dtype('>i4'),
    np.dtype('<f4'),
    np.dtype('>f4'),
    np.dtype('<f8'),
    np.dtype('>f8'),
]

@dataclass
class     __a:






    """simple docstring"""





    lowerCAmelCase         = True
    lowerCAmelCase         = None
    # Automatically constructed
    lowerCAmelCase         = "PIL.Image.Image"
    lowerCAmelCase         = pa.struct({'''bytes''': pa.binary(), '''path''': pa.string()}  )
    lowerCAmelCase         = field(default='''Image'''							,		init=_a							,		repr=_a  )





    def __call__( self    )					->						Tuple:
         return self.pa_type





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						dict:
         if config.PIL_AVAILABLE:
              import PIL.Image
         else:
              raise ImportError('''To support encoding images, please install \'Pillow\'.'''    )

         if isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              UpperCAmelCase_     :				List[str]  =     np.array(_SCREAMING_SNAKE_CASE    )

         if isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              return {"path": value, "bytes": None}
         elif isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              return {"path": None, "bytes": value}
         elif isinstance(_SCREAMING_SNAKE_CASE  ,np.ndarray    ):
              # convert the image array to PNG/TIFF bytes
              return encode_np_array(_SCREAMING_SNAKE_CASE    )
         elif isinstance(_SCREAMING_SNAKE_CASE  ,PIL.Image.Image    ):
              # convert the PIL image to bytes (default format is PNG/TIFF)
              return encode_pil_image(_SCREAMING_SNAKE_CASE    )
         elif value.get('''path'''    ) is not None and os.path.isfile(value['''path''']    ):
              # we set "bytes": None to not duplicate the data if they're already available locally
              return {"bytes": None, "path": value.get('''path'''    )}
         elif value.get('''bytes'''    ) is not None or value.get('''path'''    ) is not None:
              # store the image bytes, and path is used to infer the image format using the file extension
              return {"bytes": value.get('''bytes'''    ), "path": value.get('''path'''    )}
         else:
              raise ValueError(
                  f'''An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''    )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE=None    )					->						"PIL.Image.Image":
         if not self.decode:
              raise RuntimeError('''Decoding is disabled for this feature. Please use Image(decode=True) instead.'''    )

         if config.PIL_AVAILABLE:
              import PIL.Image
         else:
              raise ImportError('''To support decoding images, please install \'Pillow\'.'''    )

         if token_per_repo_id is None:
              UpperCAmelCase_     :				Dict  =     {}

         UpperCAmelCase_,       UpperCAmelCase_     :				Union[str, Any]  =     value['''path'''], value['''bytes''']
         if bytes_ is None:
              if path is None:
                   raise ValueError(f'''An image should have one of \'path\' or \'bytes\' but both are None in {value}.'''    )
              else:
                   if is_local_path(_SCREAMING_SNAKE_CASE    ):
                        UpperCAmelCase_     :				Tuple  =     PIL.Image.open(_SCREAMING_SNAKE_CASE    )
                   else:
                        UpperCAmelCase_     :				Dict  =     path.split('''::'''    )[-1]
                        try:
                             UpperCAmelCase_     :				Optional[int]  =     string_to_dict(_SCREAMING_SNAKE_CASE  ,config.HUB_DATASETS_URL    )['''repo_id''']
                             UpperCAmelCase_     :				Tuple  =     token_per_repo_id.get(_SCREAMING_SNAKE_CASE    )
                        except ValueError:
                             UpperCAmelCase_     :				Optional[Any]  =     None
                        with xopen(_SCREAMING_SNAKE_CASE  ,'''rb'''  ,use_auth_token=_SCREAMING_SNAKE_CASE    ) as f:
                             UpperCAmelCase_     :				List[str]  =     BytesIO(f.read()    )
                        UpperCAmelCase_     :				Optional[Any]  =     PIL.Image.open(bytes_    )
         else:
              UpperCAmelCase_     :				List[Any]  =     PIL.Image.open(BytesIO(bytes_    )    )
         image.load()  # to avoid "Too many open files" errors
         return image





    def 				a__ ( self    )					->						Union["FeatureType", Dict[str, "FeatureType"]]:
         from .features import Value

         return (
             self
             if self.decode
             else {
                 "bytes": Value('''binary'''    ),
                 "path": Value('''string'''    ),
             }
         )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						pa.StructArray:
         if pa.types.is_string(storage.type    ):
              UpperCAmelCase_     :				Dict  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.binary()    )
              UpperCAmelCase_     :				Dict  =     pa.StructArray.from_arrays([bytes_array, storage]  ,['''bytes''', '''path''']  ,mask=storage.is_null()    )
         elif pa.types.is_binary(storage.type    ):
              UpperCAmelCase_     :				List[str]  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.string()    )
              UpperCAmelCase_     :				Tuple  =     pa.StructArray.from_arrays([storage, path_array]  ,['''bytes''', '''path''']  ,mask=storage.is_null()    )
         elif pa.types.is_struct(storage.type    ):
              if storage.type.get_field_index('''bytes'''    ) >= 0:
                   UpperCAmelCase_     :				Dict  =     storage.field('''bytes'''    )
              else:
                   UpperCAmelCase_     :				Any  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.binary()    )
              if storage.type.get_field_index('''path'''    ) >= 0:
                   UpperCAmelCase_     :				int  =     storage.field('''path'''    )
              else:
                   UpperCAmelCase_     :				List[str]  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.string()    )
              UpperCAmelCase_     :				Optional[Any]  =     pa.StructArray.from_arrays([bytes_array, path_array]  ,['''bytes''', '''path''']  ,mask=storage.is_null()    )
         elif pa.types.is_list(storage.type    ):
              UpperCAmelCase_     :				Optional[Any]  =     pa.array(
                  [encode_np_array(np.array(_SCREAMING_SNAKE_CASE    )    )['''bytes'''] if arr is not None else None for arr in storage.to_pylist()]  ,type=pa.binary()  ,)
              UpperCAmelCase_     :				Any  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.string()    )
              UpperCAmelCase_     :				Dict  =     pa.StructArray.from_arrays(
                  [bytes_array, path_array]  ,['''bytes''', '''path''']  ,mask=bytes_array.is_null()    )
         return array_cast(_SCREAMING_SNAKE_CASE  ,self.pa_type    )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						pa.StructArray:

         @no_op_if_value_is_null
         def path_to_bytes(_SCREAMING_SNAKE_CASE    ):
              with xopen(_SCREAMING_SNAKE_CASE  ,'''rb'''    ) as f:
                   UpperCAmelCase_     :				Any  =     f.read()
              return bytes_

         UpperCAmelCase_     :				Union[str, Any]  =     pa.array(
             [
                 (path_to_bytes(x['''path''']    ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
                 for x in storage.to_pylist()
             ]  ,type=pa.binary()  ,)
         UpperCAmelCase_     :				List[str]  =     pa.array(
             [os.path.basename(_SCREAMING_SNAKE_CASE    ) if path is not None else None for path in storage.field('''path'''    ).to_pylist()]  ,type=pa.string()  ,)
         UpperCAmelCase_     :				Union[str, Any]  =     pa.StructArray.from_arrays([bytes_array, path_array]  ,['''bytes''', '''path''']  ,mask=bytes_array.is_null()    )
         return array_cast(_SCREAMING_SNAKE_CASE  ,self.pa_type    )







def lowerCamelCase__  (     ):

     '''simple docstring'''


     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     global _IMAGE_COMPRESSION_FORMATS
     if _IMAGE_COMPRESSION_FORMATS is None:
          PIL.Image.init()
          UpperCAmelCase_     :				Optional[int]  =     list(set(PIL.Image.OPEN.keys() ) & set(PIL.Image.SAVE.keys() ) )
     return _IMAGE_COMPRESSION_FORMATS







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				Optional[int]  =     BytesIO()
     if image.format in list_image_compression_formats():
          UpperCAmelCase_     :				int  =     image.format
     else:
          UpperCAmelCase_     :				List[Any]  =     '''PNG''' if image.mode in ['''1''', '''L''', '''LA''', '''RGB''', '''RGBA'''] else '''TIFF'''
     image.save(_lowercase					,		format=_lowercase )
     return buffer.getvalue()







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if hasattr(_lowercase					,		'''filename''' ) and image.filename != "":
          return {"path": image.filename, "bytes": None}
     else:
          return {"path": None, "bytes": image_to_bytes(_lowercase )}







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     UpperCAmelCase_     :				Tuple  =     array.dtype
     UpperCAmelCase_     :				List[str]  =     dtype.byteorder if dtype.byteorder != '''=''' else _NATIVE_BYTEORDER
     UpperCAmelCase_     :				Dict  =     dtype.kind
     UpperCAmelCase_     :				Union[str, Any]  =     dtype.itemsize

     UpperCAmelCase_     :				Optional[Any]  =     None

     # Multi-channel array case (only np.dtype("|u1") is allowed)
     if array.shape[2:]:
          UpperCAmelCase_     :				Tuple  =     np.dtype('''|u1''' )
          if dtype_kind not in ["u", "i"]:
               raise TypeError(
                   f'''Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.''' )
          if dtype is not dest_dtype:
               warnings.warn(f'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\'''' )
    # Exact match
     elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
          UpperCAmelCase_     :				Union[str, Any]  =     dtype
     else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
          while dtype_itemsize >= 1:
               UpperCAmelCase_     :				Union[str, Any]  =     dtype_byteorder + dtype_kind + str(_lowercase )
               UpperCAmelCase_     :				str  =     np.dtype(_lowercase )
               if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
                    warnings.warn(f'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\'''' )
                    break
               else:
                    dtype_itemsize //= 2

     if dest_dtype is None:
          raise TypeError(
              f'''Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}''' )
     UpperCAmelCase_     :				Any  =     PIL.Image.fromarray(array.astype(_lowercase ) )
     return {"path": None, "bytes": image_to_bytes(_lowercase )}







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     if objs:
          UpperCAmelCase_,       UpperCAmelCase_     :				Tuple  =     first_non_null_value(_lowercase )
          if isinstance(_lowercase					,		_lowercase ):
               return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
          if isinstance(_lowercase					,		np.ndarray ):
               UpperCAmelCase_     :				Any  =     no_op_if_value_is_null(_lowercase )
               return [obj_to_image_dict_func(_lowercase ) for obj in objs]
          elif isinstance(_lowercase					,		PIL.Image.Image ):
               UpperCAmelCase_     :				Union[str, Any]  =     no_op_if_value_is_null(_lowercase )
               return [obj_to_image_dict_func(_lowercase ) for obj in objs]
          else:
               return objs
     else:
          return objs