





'''simple docstring'''

import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotConfig, is_flax_available
from transformers.testing_utils import jax_device, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
   import os

   # The slow tests are often failing with OOM error on GPU
   # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
   # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
   lowercase__		:     Any			=							'platform'

   import jax
   import jax.numpy as jnp

   from transformers import BlenderbotTokenizer
   from transformers.models.blenderbot.modeling_flax_blenderbot import (
       FlaxBlenderbotForConditionalGeneration,
       FlaxBlenderbotModel,
       shift_tokens_right,
   )







def    a__							(							lowercase   :	Tuple,   lowercase   :	Optional[int],   lowercase   :	Any=None,   lowercase   :	Optional[Any]=None,   lowercase   :	Dict=None,   lowercase   :	List[str]=None,   lowercase   :	Any=None,   lowercase   :	Tuple=None,   )						->						Tuple:





     """simple docstring"""

     if attention_mask is None:
          _UpperCamelCase							       =			np.where(input_ids != config.pad_token_id,   1,   0 )
     if decoder_attention_mask is None:
          _UpperCamelCase							       =			np.where(decoder_input_ids != config.pad_token_id,   1,   0 )
     if head_mask is None:
          _UpperCamelCase							       =			np.ones((config.encoder_layers, config.encoder_attention_heads) )
     if decoder_head_mask is None:
          _UpperCamelCase							       =			np.ones((config.decoder_layers, config.decoder_attention_heads) )
     if cross_attn_head_mask is None:
          _UpperCamelCase							       =			np.ones((config.decoder_layers, config.decoder_attention_heads) )
     return {
         "input_ids": input_ids,
         "decoder_input_ids": decoder_input_ids,
         "attention_mask": attention_mask,
         "decoder_attention_mask": attention_mask,
     }
class 							__lowerCAmelCase	:






       """simple docstring"""





       def __init__(				self  :						int			,					lowerCAmelCase__  :						Optional[Any]			,					lowerCAmelCase__  :						List[str]=13			,					lowerCAmelCase__  :						str=7			,					lowerCAmelCase__  :						Optional[Any]=True			,					lowerCAmelCase__  :						Tuple=False			,					lowerCAmelCase__  :						Tuple=99			,					lowerCAmelCase__  :						Dict=16			,					lowerCAmelCase__  :						List[str]=2			,					lowerCAmelCase__  :						int=4			,					lowerCAmelCase__  :						Tuple=4			,					lowerCAmelCase__  :						Dict="gelu"			,					lowerCAmelCase__  :						Optional[Any]=0.1			,					lowerCAmelCase__  :						Union[str, Any]=0.1			,					lowerCAmelCase__  :						Dict=32			,					lowerCAmelCase__  :						str=2			,					lowerCAmelCase__  :						Tuple=1			,					lowerCAmelCase__  :						Tuple=0			,					lowerCAmelCase__  :						List[Any]=0.02			,					)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			parent
            _UpperCamelCase							       =			batch_size
            _UpperCamelCase							       =			seq_length
            _UpperCamelCase							       =			is_training
            _UpperCamelCase							       =			use_labels
            _UpperCamelCase							       =			vocab_size
            _UpperCamelCase							       =			hidden_size
            _UpperCamelCase							       =			num_hidden_layers
            _UpperCamelCase							       =			num_attention_heads
            _UpperCamelCase							       =			intermediate_size
            _UpperCamelCase							       =			hidden_act
            _UpperCamelCase							       =			hidden_dropout_prob
            _UpperCamelCase							       =			attention_probs_dropout_prob
            _UpperCamelCase							       =			max_position_embeddings
            _UpperCamelCase							       =			eos_token_id
            _UpperCamelCase							       =			pad_token_id
            _UpperCamelCase							       =			bos_token_id
            _UpperCamelCase							       =			initializer_range





       def snake_case__		(				self  :						List[str]	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			np.clip(ids_tensor([self.batch_size, self.seq_length - 1]			,					self.vocab_size	)			,					3			,					self.vocab_size	)
            _UpperCamelCase							       =			np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)			,					dtype=np.intaa	))			,					-1	)

            _UpperCamelCase							       =			shift_tokens_right(lowerCAmelCase__			,					1			,					2	)

            _UpperCamelCase							       =			BlenderbotConfig(
                vocab_size=self.vocab_size			,					d_model=self.hidden_size			,					encoder_layers=self.num_hidden_layers			,					decoder_layers=self.num_hidden_layers			,					encoder_attention_heads=self.num_attention_heads			,					decoder_attention_heads=self.num_attention_heads			,					encoder_ffn_dim=self.intermediate_size			,					decoder_ffn_dim=self.intermediate_size			,					dropout=self.hidden_dropout_prob			,					attention_dropout=self.attention_probs_dropout_prob			,					max_position_embeddings=self.max_position_embeddings			,					eos_token_id=self.eos_token_id			,					bos_token_id=self.bos_token_id			,					pad_token_id=self.pad_token_id			,					initializer_range=self.initializer_range			,					use_cache=lowerCAmelCase__			,					)
            _UpperCamelCase							       =			prepare_blenderbot_inputs_dict(lowerCAmelCase__			,					lowerCAmelCase__			,					lowerCAmelCase__	)
            return config, inputs_dict





       def snake_case__		(				self  :						int	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase       ,							_UpperCamelCase							       =			self.prepare_config_and_inputs()
            return config, inputs_dict





       def snake_case__		(				self  :						str			,					lowerCAmelCase__  :						List[str]			,					lowerCAmelCase__  :						Tuple			,					lowerCAmelCase__  :						Tuple	)       ->	Tuple:

            '''simple docstring'''




            _UpperCamelCase							       =			20
            _UpperCamelCase							       =			model_class_name(lowerCAmelCase__	)

            _UpperCamelCase							       =			model.encode(inputs_dict['''input_ids''']	)

            _UpperCamelCase       ,							_UpperCamelCase							       =			(
                inputs_dict['''decoder_input_ids'''],
                inputs_dict['''decoder_attention_mask'''],
            )

            _UpperCamelCase							       =			model.init_cache(decoder_input_ids.shape[0]			,					lowerCAmelCase__			,					lowerCAmelCase__	)
            _UpperCamelCase							       =			jnp.ones((decoder_input_ids.shape[0], max_decoder_length)			,					dtype='''i4'''	)

            _UpperCamelCase							       =			jnp.broadcast_to(
                jnp.arange(decoder_input_ids.shape[-1] - 1	)[None, :]			,					(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)			,					)
            _UpperCamelCase							       =			model.decode(
                decoder_input_ids[:, :-1]			,					lowerCAmelCase__			,					decoder_attention_mask=lowerCAmelCase__			,					past_key_values=lowerCAmelCase__			,					decoder_position_ids=lowerCAmelCase__			,					)

            _UpperCamelCase							       =			jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]			,					dtype='''i4'''	)
            _UpperCamelCase							       =			model.decode(
                decoder_input_ids[:, -1:]			,					lowerCAmelCase__			,					decoder_attention_mask=lowerCAmelCase__			,					past_key_values=outputs_cache.past_key_values			,					decoder_position_ids=lowerCAmelCase__			,					)

            _UpperCamelCase							       =			model.decode(lowerCAmelCase__			,					lowerCAmelCase__	)

            _UpperCamelCase							       =			np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
            self.parent.assertTrue(diff < 1e-3			,					msg=f"""Max diff is {diff}"""	)







       def snake_case__		(				self  :						int			,					lowerCAmelCase__  :						Optional[Any]			,					lowerCAmelCase__  :						Optional[int]			,					lowerCAmelCase__  :						Optional[int]	)       ->	Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			20
            _UpperCamelCase							       =			model_class_name(lowerCAmelCase__	)

            _UpperCamelCase							       =			model.encode(inputs_dict['''input_ids''']	)

            _UpperCamelCase       ,							_UpperCamelCase							       =			(
                inputs_dict['''decoder_input_ids'''],
                inputs_dict['''decoder_attention_mask'''],
            )

            _UpperCamelCase							       =			jnp.concatenate(
                [
                    decoder_attention_mask,
                    jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])	),
                ]			,					axis=-1			,					)

            _UpperCamelCase							       =			model.init_cache(decoder_input_ids.shape[0]			,					lowerCAmelCase__			,					lowerCAmelCase__	)
            _UpperCamelCase							       =			jnp.broadcast_to(
                jnp.arange(decoder_input_ids.shape[-1] - 1	)[None, :]			,					(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)			,					)

            _UpperCamelCase							       =			model.decode(
                decoder_input_ids[:, :-1]			,					lowerCAmelCase__			,					decoder_attention_mask=lowerCAmelCase__			,					past_key_values=lowerCAmelCase__			,					decoder_position_ids=lowerCAmelCase__			,					)
            _UpperCamelCase							       =			jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]			,					dtype='''i4'''	)
            _UpperCamelCase							       =			model.decode(
                decoder_input_ids[:, -1:]			,					lowerCAmelCase__			,					past_key_values=outputs_cache.past_key_values			,					decoder_attention_mask=lowerCAmelCase__			,					decoder_position_ids=lowerCAmelCase__			,					)

            _UpperCamelCase							       =			model.decode(lowerCAmelCase__			,					lowerCAmelCase__			,					decoder_attention_mask=lowerCAmelCase__	)

            _UpperCamelCase							       =			np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
            self.parent.assertTrue(diff < 1e-3			,					msg=f"""Max diff is {diff}"""	)

@require_flax
class 							__lowerCAmelCase	(       unittest.TestCase					):






       """simple docstring"""
       _snake_case      :	List[Any]									=       9_9





       def snake_case__		(				self  :						Dict	)       ->	List[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			np.array(
                [
                    [71, 82, 18, 33, 46, 91, 2],
                    [68, 34, 26, 58, 30, 82, 2],
                    [5, 97, 17, 39, 94, 40, 2],
                    [76, 83, 94, 25, 70, 78, 2],
                    [87, 59, 41, 35, 48, 66, 2],
                    [55, 13, 16, 58, 5, 2, 1],  # note padding
                    [64, 27, 31, 51, 12, 75, 2],
                    [52, 64, 86, 17, 83, 39, 2],
                    [48, 61, 9, 24, 71, 82, 2],
                    [26, 1, 60, 48, 22, 13, 2],
                    [21, 5, 62, 28, 14, 76, 2],
                    [45, 98, 37, 86, 59, 48, 2],
                    [70, 70, 50, 9, 28, 0, 2],
                ]			,					dtype=np.intaa			,					)

            _UpperCamelCase							       =			input_ids.shape[0]
            _UpperCamelCase							       =			BlenderbotConfig(
                vocab_size=self.vocab_size			,					d_model=24			,					encoder_layers=2			,					decoder_layers=2			,					encoder_attention_heads=2			,					decoder_attention_heads=2			,					encoder_ffn_dim=32			,					decoder_ffn_dim=32			,					max_position_embeddings=48			,					eos_token_id=2			,					pad_token_id=1			,					bos_token_id=0			,					)
            return config, input_ids, batch_size





       def snake_case__		(				self  :						Dict	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase							       =			self._get_config_and_data()
            _UpperCamelCase							       =			FlaxBlenderbotForConditionalGeneration(lowerCAmelCase__	)
            _UpperCamelCase							       =			lm_model(input_ids=lowerCAmelCase__	)
            _UpperCamelCase							       =			(batch_size, input_ids.shape[1], config.vocab_size)
            self.assertEqual(outputs['''logits'''].shape			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						Any	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			BlenderbotConfig(
                vocab_size=self.vocab_size			,					d_model=14			,					encoder_layers=2			,					decoder_layers=2			,					encoder_attention_heads=2			,					decoder_attention_heads=2			,					encoder_ffn_dim=8			,					decoder_ffn_dim=8			,					max_position_embeddings=48			,					)
            _UpperCamelCase							       =			FlaxBlenderbotForConditionalGeneration(lowerCAmelCase__	)
            _UpperCamelCase							       =			np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]			,					dtype=np.intaa	)
            _UpperCamelCase							       =			np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]			,					dtype=np.intaa	)
            _UpperCamelCase							       =			lm_model(input_ids=lowerCAmelCase__			,					decoder_input_ids=lowerCAmelCase__	)
            _UpperCamelCase							       =			(*summary.shape, config.vocab_size)
            self.assertEqual(outputs['''logits'''].shape			,					lowerCAmelCase__	)







       def snake_case__		(				self  :						List[str]	)       ->	Dict:

            '''simple docstring'''




            _UpperCamelCase							       =			np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]			,					dtype=np.intaa	)
            _UpperCamelCase							       =			shift_tokens_right(lowerCAmelCase__			,					1			,					2	)
            _UpperCamelCase							       =			np.equal(lowerCAmelCase__			,					1	).astype(np.floataa	).sum()
            _UpperCamelCase							       =			np.equal(lowerCAmelCase__			,					1	).astype(np.floataa	).sum()
            self.assertEqual(shifted.shape			,					input_ids.shape	)
            self.assertEqual(lowerCAmelCase__			,					n_pad_before - 1	)
            self.assertTrue(np.equal(shifted[:, 0]			,					2	).all()	)

@require_flax
class 							__lowerCAmelCase	(       __magic_name__		,		unittest.TestCase		,		__magic_name__					):






       """simple docstring"""
       _snake_case      :	Dict									=       True
       _snake_case      :	str									=       (
           (
               FlaxBlenderbotModel,
               FlaxBlenderbotForConditionalGeneration,
           )
           if is_flax_available()
           else ()
       )
       _snake_case      :	Tuple									=       (FlaxBlenderbotForConditionalGeneration,) if is_flax_available() else ()





       def snake_case__		(				self  :						Union[str, Any]	)       ->	List[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			FlaxBlenderbotModelTester(self	)





       def snake_case__		(				self  :						List[str]	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase       ,							_UpperCamelCase							       =			self.model_tester.prepare_config_and_inputs()
            for model_class in self.all_model_classes:
                 self.model_tester.check_use_cache_forward(lowerCAmelCase__			,					lowerCAmelCase__			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						Union[str, Any]	)       ->	Tuple:

            '''simple docstring'''




            _UpperCamelCase       ,							_UpperCamelCase							       =			self.model_tester.prepare_config_and_inputs()
            for model_class in self.all_model_classes:
                 self.model_tester.check_use_cache_forward_with_attn_mask(lowerCAmelCase__			,					lowerCAmelCase__			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						Union[str, Any]	)       ->	Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase       ,							_UpperCamelCase							       =			self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 with self.subTest(model_class.__name__	):
                      _UpperCamelCase							       =			self._prepare_for_class(lowerCAmelCase__			,					lowerCAmelCase__	)
                      _UpperCamelCase							       =			model_class(lowerCAmelCase__	)

                      @jax.jit
                      def encode_jitted(lowerCAmelCase__  :						Union[str, Any]			,					lowerCAmelCase__  :						Dict=None			,					**lowerCAmelCase__  :						Optional[Any]	):
                           return model.encode(input_ids=lowerCAmelCase__			,					attention_mask=lowerCAmelCase__	)

                      with self.subTest('''JIT Enabled'''	):
                           _UpperCamelCase							       =			encode_jitted(**lowerCAmelCase__	).to_tuple()

                      with self.subTest('''JIT Disabled'''	):
                           with jax.disable_jit():
                                _UpperCamelCase							       =			encode_jitted(**lowerCAmelCase__	).to_tuple()

                      self.assertEqual(len(lowerCAmelCase__	)			,					len(lowerCAmelCase__	)	)
                      for jitted_output, output in zip(lowerCAmelCase__			,					lowerCAmelCase__	):
                           self.assertEqual(jitted_output.shape			,					output.shape	)





       def snake_case__		(				self  :						Dict	)       ->	str:

            '''simple docstring'''




            _UpperCamelCase       ,							_UpperCamelCase							       =			self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 with self.subTest(model_class.__name__	):
                      _UpperCamelCase							       =			model_class(lowerCAmelCase__	)
                      _UpperCamelCase							       =			model.encode(inputs_dict['''input_ids''']			,					inputs_dict['''attention_mask''']	)

                      _UpperCamelCase							       =			{
                          '''decoder_input_ids''': inputs_dict['''decoder_input_ids'''],
                          '''decoder_attention_mask''': inputs_dict['''decoder_attention_mask'''],
                          '''encoder_outputs''': encoder_outputs,
                      }

                      @jax.jit
                      def decode_jitted(lowerCAmelCase__  :						List[Any]			,					lowerCAmelCase__  :						List[Any]			,					lowerCAmelCase__  :						List[Any]	):
                           return model.decode(
                               decoder_input_ids=lowerCAmelCase__			,					decoder_attention_mask=lowerCAmelCase__			,					encoder_outputs=lowerCAmelCase__			,					)

                      with self.subTest('''JIT Enabled'''	):
                           _UpperCamelCase							       =			decode_jitted(**lowerCAmelCase__	).to_tuple()

                      with self.subTest('''JIT Disabled'''	):
                           with jax.disable_jit():
                                _UpperCamelCase							       =			decode_jitted(**lowerCAmelCase__	).to_tuple()

                      self.assertEqual(len(lowerCAmelCase__	)			,					len(lowerCAmelCase__	)	)
                      for jitted_output, output in zip(lowerCAmelCase__			,					lowerCAmelCase__	):
                           self.assertEqual(jitted_output.shape			,					output.shape	)





       @slow
       def snake_case__		(				self  :						Dict	)       ->	Any:

            '''simple docstring'''




            for model_class_name in self.all_model_classes:
                 _UpperCamelCase							       =			model_class_name.from_pretrained('''facebook/blenderbot-400M-distill'''	)
                 # FlaxBlenderbotForSequenceClassification expects eos token in input_ids
                 _UpperCamelCase							       =			np.ones((1, 1)	) * model.config.eos_token_id
                 _UpperCamelCase							       =			model(lowerCAmelCase__	)
                 self.assertIsNotNone(lowerCAmelCase__	)







       @unittest.skipUnless(jax_device != '''cpu'''			,					'''3B test too slow on CPU.'''	)
       @slow
       def snake_case__		(				self  :						int	)       ->	Tuple:

            '''simple docstring'''




            _UpperCamelCase							       =			{'''num_beams''': 1, '''early_stopping''': True, '''min_length''': 15, '''max_length''': 25}
            _UpperCamelCase							       =			{'''skip_special_tokens''': True, '''clean_up_tokenization_spaces''': True}

            _UpperCamelCase							       =			FlaxBlenderbotForConditionalGeneration.from_pretrained('''facebook/blenderbot-3B'''			,					from_pt=lowerCAmelCase__	)
            _UpperCamelCase							       =			BlenderbotTokenizer.from_pretrained('''facebook/blenderbot-3B'''	)

            _UpperCamelCase							       =			['''Sam''']
            _UpperCamelCase							       =			tokenizer(lowerCAmelCase__			,					return_tensors='''jax'''	)

            _UpperCamelCase							       =			model.generate(**lowerCAmelCase__			,					**lowerCAmelCase__	)
            _UpperCamelCase							       =			'''Sam is a great name. It means "sun" in Gaelic.'''

            _UpperCamelCase							       =			tokenizer.batch_decode(lowerCAmelCase__			,					**lowerCAmelCase__	)
            assert generated_txt[0].strip() == tgt_text










'''simple docstring'''

import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
   import PIL.Image

   from .features import FeatureType


lowercase__		:     Optional[List[str]]			=							None
lowercase__		:     List[Any]			=							'<' if sys.byteorder == 'little' else '>'
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
lowercase__		:     int			=							[
    np.dtype('|b1'),
    np.dtype('|u1'),
    np.dtype('<u2'),
    np.dtype('>u2'),
    np.dtype('<i2'),
    np.dtype('>i2'),
    np.dtype('<u4'),
    np.dtype('>u4'),
    np.dtype('<i4'),
    np.dtype('>i4'),
    np.dtype('<f4'),
    np.dtype('>f4'),
    np.dtype('<f8'),
    np.dtype('>f8'),
]
@dataclass
class 							__lowerCAmelCase	:






       """simple docstring"""
       _snake_case      :	bool									=       True
       _snake_case      :	Optional[str]									=       None
       # Automatically constructed
       _snake_case      :	ClassVar[str]									=       "PIL.Image.Image"
       _snake_case      :	ClassVar[Any]									=       pa.struct({'bytes': pa.binary(), 'path': pa.string()}					)
       _snake_case      :	str									=       field(default='Image'		,		init=__magic_name__		,		repr=__magic_name__					)





       def __call__(				self  :						Any	)       ->	str:

            '''simple docstring'''




            return self.pa_type





       def snake_case__		(				self  :						Dict			,					lowerCAmelCase__  :						Union[str, bytes, dict, np.ndarray, "PIL.Image.Image"]	)       ->	dict:

            '''simple docstring'''




            if config.PIL_AVAILABLE:
                 import PIL.Image
            else:
                 raise ImportError('''To support encoding images, please install \'Pillow\'.'''	)

            if isinstance(lowerCAmelCase__			,					lowerCAmelCase__	):
                 _UpperCamelCase							       =			np.array(lowerCAmelCase__	)

            if isinstance(lowerCAmelCase__			,					lowerCAmelCase__	):
                 return {"path": value, "bytes": None}
            elif isinstance(lowerCAmelCase__			,					lowerCAmelCase__	):
                 return {"path": None, "bytes": value}
            elif isinstance(lowerCAmelCase__			,					np.ndarray	):
                 # convert the image array to PNG/TIFF bytes
                 return encode_np_array(lowerCAmelCase__	)
            elif isinstance(lowerCAmelCase__			,					PIL.Image.Image	):
                 # convert the PIL image to bytes (default format is PNG/TIFF)
                 return encode_pil_image(lowerCAmelCase__	)
            elif value.get('''path'''	) is not None and os.path.isfile(value['''path''']	):
                 # we set "bytes": None to not duplicate the data if they're already available locally
                 return {"bytes": None, "path": value.get('''path'''	)}
            elif value.get('''bytes'''	) is not None or value.get('''path'''	) is not None:
                 # store the image bytes, and path is used to infer the image format using the file extension
                 return {"bytes": value.get('''bytes'''	), "path": value.get('''path'''	)}
            else:
                 raise ValueError(
                     f"""An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."""	)





       def snake_case__		(				self  :						Tuple			,					lowerCAmelCase__  :						dict			,					lowerCAmelCase__  :						Dict=None	)       ->	"PIL.Image.Image":

            '''simple docstring'''




            if not self.decode:
                 raise RuntimeError('''Decoding is disabled for this feature. Please use Image(decode=True) instead.'''	)

            if config.PIL_AVAILABLE:
                 import PIL.Image
            else:
                 raise ImportError('''To support decoding images, please install \'Pillow\'.'''	)

            if token_per_repo_id is None:
                 _UpperCamelCase							       =			{}

            _UpperCamelCase       ,							_UpperCamelCase							       =			value['''path'''], value['''bytes''']
            if bytes_ is None:
                 if path is None:
                      raise ValueError(f"""An image should have one of 'path' or 'bytes' but both are None in {value}."""	)
                 else:
                      if is_local_path(lowerCAmelCase__	):
                           _UpperCamelCase							       =			PIL.Image.open(lowerCAmelCase__	)
                      else:
                           _UpperCamelCase							       =			path.split('''::'''	)[-1]
                           try:
                                _UpperCamelCase							       =			string_to_dict(lowerCAmelCase__			,					config.HUB_DATASETS_URL	)['''repo_id''']
                                _UpperCamelCase							       =			token_per_repo_id.get(lowerCAmelCase__	)
                           except ValueError:
                                _UpperCamelCase							       =			None
                           with xopen(lowerCAmelCase__			,					'''rb'''			,					use_auth_token=lowerCAmelCase__	) as f:
                                _UpperCamelCase							       =			BytesIO(f.read()	)
                           _UpperCamelCase							       =			PIL.Image.open(bytes_	)
            else:
                 _UpperCamelCase							       =			PIL.Image.open(BytesIO(bytes_	)	)
            image.load()  # to avoid "Too many open files" errors
            return image





       def snake_case__		(				self  :						List[str]	)       ->	Union["FeatureType", Dict[str, "FeatureType"]]:

            '''simple docstring'''




            from .features import Value

            return (
                self
                if self.decode
                else {
                    "bytes": Value('''binary'''	),
                    "path": Value('''string'''	),
                }
            )





       def snake_case__		(				self  :						int			,					lowerCAmelCase__  :						Union[pa.StringArray, pa.StructArray, pa.ListArray]	)       ->	pa.StructArray:

            '''simple docstring'''




            if pa.types.is_string(storage.type	):
                 _UpperCamelCase							       =			pa.array([None] * len(lowerCAmelCase__	)			,					type=pa.binary()	)
                 _UpperCamelCase							       =			pa.StructArray.from_arrays([bytes_array, storage]			,					['''bytes''', '''path''']			,					mask=storage.is_null()	)
            elif pa.types.is_binary(storage.type	):
                 _UpperCamelCase							       =			pa.array([None] * len(lowerCAmelCase__	)			,					type=pa.string()	)
                 _UpperCamelCase							       =			pa.StructArray.from_arrays([storage, path_array]			,					['''bytes''', '''path''']			,					mask=storage.is_null()	)
            elif pa.types.is_struct(storage.type	):
                 if storage.type.get_field_index('''bytes'''	) >= 0:
                      _UpperCamelCase							       =			storage.field('''bytes'''	)
                 else:
                      _UpperCamelCase							       =			pa.array([None] * len(lowerCAmelCase__	)			,					type=pa.binary()	)
                 if storage.type.get_field_index('''path'''	) >= 0:
                      _UpperCamelCase							       =			storage.field('''path'''	)
                 else:
                      _UpperCamelCase							       =			pa.array([None] * len(lowerCAmelCase__	)			,					type=pa.string()	)
                 _UpperCamelCase							       =			pa.StructArray.from_arrays([bytes_array, path_array]			,					['''bytes''', '''path''']			,					mask=storage.is_null()	)
            elif pa.types.is_list(storage.type	):
                 _UpperCamelCase							       =			pa.array(
                     [encode_np_array(np.array(lowerCAmelCase__	)	)['''bytes'''] if arr is not None else None for arr in storage.to_pylist()]			,					type=pa.binary()			,					)
                 _UpperCamelCase							       =			pa.array([None] * len(lowerCAmelCase__	)			,					type=pa.string()	)
                 _UpperCamelCase							       =			pa.StructArray.from_arrays(
                     [bytes_array, path_array]			,					['''bytes''', '''path''']			,					mask=bytes_array.is_null()	)
            return array_cast(lowerCAmelCase__			,					self.pa_type	)







       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						pa.StructArray	)       ->	pa.StructArray:

            '''simple docstring'''




            @no_op_if_value_is_null
            def path_to_bytes(lowerCAmelCase__  :						Optional[Any]	):
                 with xopen(lowerCAmelCase__			,					'''rb'''	) as f:
                      _UpperCamelCase							       =			f.read()
                 return bytes_

            _UpperCamelCase							       =			pa.array(
                [
                    (path_to_bytes(x['''path''']	) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
                    for x in storage.to_pylist()
                ]			,					type=pa.binary()			,					)
            _UpperCamelCase							       =			pa.array(
                [os.path.basename(lowerCAmelCase__	) if path is not None else None for path in storage.field('''path'''	).to_pylist()]			,					type=pa.string()			,					)
            _UpperCamelCase							       =			pa.StructArray.from_arrays([bytes_array, path_array]			,					['''bytes''', '''path''']			,					mask=bytes_array.is_null()	)
            return array_cast(lowerCAmelCase__			,					self.pa_type	)







def    a__							(							)						->						List[str]:





     """simple docstring"""

     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     global _IMAGE_COMPRESSION_FORMATS
     if _IMAGE_COMPRESSION_FORMATS is None:
          PIL.Image.init()
          _UpperCamelCase							       =			list(set(PIL.Image.OPEN.keys() ) & set(PIL.Image.SAVE.keys() ) )
     return _IMAGE_COMPRESSION_FORMATS







def    a__							(							lowercase   :	"PIL.Image.Image" )						->						bytes:





     """simple docstring"""

     _UpperCamelCase							       =			BytesIO()
     if image.format in list_image_compression_formats():
          _UpperCamelCase							       =			image.format
     else:
          _UpperCamelCase							       =			'''PNG''' if image.mode in ['''1''', '''L''', '''LA''', '''RGB''', '''RGBA'''] else '''TIFF'''
     image.save(lowercase,   format=lowercase )
     return buffer.getvalue()







def    a__							(							lowercase   :	"PIL.Image.Image" )						->						dict:





     """simple docstring"""

     if hasattr(lowercase,   '''filename''' ) and image.filename != "":
          return {"path": image.filename, "bytes": None}
     else:
          return {"path": None, "bytes": image_to_bytes(lowercase )}







def    a__							(							lowercase   :	np.ndarray )						->						dict:





     """simple docstring"""

     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     _UpperCamelCase							       =			array.dtype
     _UpperCamelCase							       =			dtype.byteorder if dtype.byteorder != '''=''' else _NATIVE_BYTEORDER
     _UpperCamelCase							       =			dtype.kind
     _UpperCamelCase							       =			dtype.itemsize

     _UpperCamelCase							       =			None

     # Multi-channel array case (only np.dtype("|u1") is allowed)
     if array.shape[2:]:
          _UpperCamelCase							       =			np.dtype('''|u1''' )
          if dtype_kind not in ["u", "i"]:
               raise TypeError(
                   F"""Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.""" )
          if dtype is not dest_dtype:
               warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'""" )
    # Exact match
     elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
          _UpperCamelCase							       =			dtype
     else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
          while dtype_itemsize >= 1:
               _UpperCamelCase							       =			dtype_byteorder + dtype_kind + str(lowercase )
               _UpperCamelCase							       =			np.dtype(lowercase )
               if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
                    warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'""" )
                    break
               else:
                    dtype_itemsize //= 2

     if dest_dtype is None:
          raise TypeError(
              F"""Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}""" )
     _UpperCamelCase							       =			PIL.Image.fromarray(array.astype(lowercase ) )
     return {"path": None, "bytes": image_to_bytes(lowercase )}







def    a__							(							lowercase   :	Union[List[str], List[dict], List[np.ndarray], List["PIL.Image.Image"]] )						->						List[dict]:





     """simple docstring"""

     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     if objs:
          _UpperCamelCase       ,							_UpperCamelCase							       =			first_non_null_value(lowercase )
          if isinstance(lowercase,   lowercase ):
               return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
          if isinstance(lowercase,   np.ndarray ):
               _UpperCamelCase							       =			no_op_if_value_is_null(lowercase )
               return [obj_to_image_dict_func(lowercase ) for obj in objs]
          elif isinstance(lowercase,   PIL.Image.Image ):
               _UpperCamelCase							       =			no_op_if_value_is_null(lowercase )
               return [obj_to_image_dict_func(lowercase ) for obj in objs]
          else:
               return objs
     else:
          return objs



