




import argparse

import torch

from transformers import (
    EncodecConfig,
    EncodecFeatureExtractor,
    EncodecModel,
    logging,
)


# checkpoints downloaded from:
# https://dl.fbaipublicfiles.com/encodec/v0/encodec_24khz-d7cc33bc.th
# https://huggingface.co/facebook/musicgen-small/resolve/main/compression_state_dict.bin
# https://dl.fbaipublicfiles.com/encodec/v0/encodec_48khz-7e698e3e.th


logging.set_verbosity_info()
__a							     =					logging.get_logger('transformers.models.encodec')

__a							     =					{
    'quantizer.vq.layers.*._codebook.inited': 'quantizer.layers.*.codebook.inited',
    'quantizer.vq.layers.*._codebook.cluster_size': 'quantizer.layers.*.codebook.cluster_size',
    'quantizer.vq.layers.*._codebook.embed': 'quantizer.layers.*.codebook.embed',
    'quantizer.vq.layers.*._codebook.embed_avg': 'quantizer.layers.*.codebook.embed_avg',
}
__a							     =					{
    'encoder.model.0.conv.conv': 'encoder.layers.0.conv',
    'encoder.model.1.block.1.conv.conv': 'encoder.layers.1.block.1.conv',
    'encoder.model.1.block.3.conv.conv': 'encoder.layers.1.block.3.conv',
    'encoder.model.1.shortcut.conv.conv': 'encoder.layers.1.shortcut.conv',
    'encoder.model.3.conv.conv': 'encoder.layers.3.conv',
    'encoder.model.4.block.1.conv.conv': 'encoder.layers.4.block.1.conv',
    'encoder.model.4.block.3.conv.conv': 'encoder.layers.4.block.3.conv',
    'encoder.model.4.shortcut.conv.conv': 'encoder.layers.4.shortcut.conv',
    'encoder.model.6.conv.conv': 'encoder.layers.6.conv',
    'encoder.model.7.block.1.conv.conv': 'encoder.layers.7.block.1.conv',
    'encoder.model.7.block.3.conv.conv': 'encoder.layers.7.block.3.conv',
    'encoder.model.7.shortcut.conv.conv': 'encoder.layers.7.shortcut.conv',
    'encoder.model.9.conv.conv': 'encoder.layers.9.conv',
    'encoder.model.10.block.1.conv.conv': 'encoder.layers.10.block.1.conv',
    'encoder.model.10.block.3.conv.conv': 'encoder.layers.10.block.3.conv',
    'encoder.model.10.shortcut.conv.conv': 'encoder.layers.10.shortcut.conv',
    'encoder.model.12.conv.conv': 'encoder.layers.12.conv',
    'encoder.model.13.lstm': 'encoder.layers.13.lstm',
    'encoder.model.15.conv.conv': 'encoder.layers.15.conv',
}
__a							     =					{
    'encoder.model.0.conv.norm': 'encoder.layers.0.norm',
    'encoder.model.1.block.1.conv.norm': 'encoder.layers.1.block.1.norm',
    'encoder.model.1.block.3.conv.norm': 'encoder.layers.1.block.3.norm',
    'encoder.model.1.shortcut.conv.norm': 'encoder.layers.1.shortcut.norm',
    'encoder.model.3.conv.norm': 'encoder.layers.3.norm',
    'encoder.model.4.block.1.conv.norm': 'encoder.layers.4.block.1.norm',
    'encoder.model.4.block.3.conv.norm': 'encoder.layers.4.block.3.norm',
    'encoder.model.4.shortcut.conv.norm': 'encoder.layers.4.shortcut.norm',
    'encoder.model.6.conv.norm': 'encoder.layers.6.norm',
    'encoder.model.7.block.1.conv.norm': 'encoder.layers.7.block.1.norm',
    'encoder.model.7.block.3.conv.norm': 'encoder.layers.7.block.3.norm',
    'encoder.model.7.shortcut.conv.norm': 'encoder.layers.7.shortcut.norm',
    'encoder.model.9.conv.norm': 'encoder.layers.9.norm',
    'encoder.model.10.block.1.conv.norm': 'encoder.layers.10.block.1.norm',
    'encoder.model.10.block.3.conv.norm': 'encoder.layers.10.block.3.norm',
    'encoder.model.10.shortcut.conv.norm': 'encoder.layers.10.shortcut.norm',
    'encoder.model.12.conv.norm': 'encoder.layers.12.norm',
    'encoder.model.15.conv.norm': 'encoder.layers.15.norm',
}
__a							     =					{
    'decoder.model.0.conv.conv': 'decoder.layers.0.conv',
    'decoder.model.1.lstm': 'decoder.layers.1.lstm',
    'decoder.model.3.convtr.convtr': 'decoder.layers.3.conv',
    'decoder.model.4.block.1.conv.conv': 'decoder.layers.4.block.1.conv',
    'decoder.model.4.block.3.conv.conv': 'decoder.layers.4.block.3.conv',
    'decoder.model.4.shortcut.conv.conv': 'decoder.layers.4.shortcut.conv',
    'decoder.model.6.convtr.convtr': 'decoder.layers.6.conv',
    'decoder.model.7.block.1.conv.conv': 'decoder.layers.7.block.1.conv',
    'decoder.model.7.block.3.conv.conv': 'decoder.layers.7.block.3.conv',
    'decoder.model.7.shortcut.conv.conv': 'decoder.layers.7.shortcut.conv',
    'decoder.model.9.convtr.convtr': 'decoder.layers.9.conv',
    'decoder.model.10.block.1.conv.conv': 'decoder.layers.10.block.1.conv',
    'decoder.model.10.block.3.conv.conv': 'decoder.layers.10.block.3.conv',
    'decoder.model.10.shortcut.conv.conv': 'decoder.layers.10.shortcut.conv',
    'decoder.model.12.convtr.convtr': 'decoder.layers.12.conv',
    'decoder.model.13.block.1.conv.conv': 'decoder.layers.13.block.1.conv',
    'decoder.model.13.block.3.conv.conv': 'decoder.layers.13.block.3.conv',
    'decoder.model.13.shortcut.conv.conv': 'decoder.layers.13.shortcut.conv',
    'decoder.model.15.conv.conv': 'decoder.layers.15.conv',
}
__a							     =					{
    'decoder.model.0.conv.norm': 'decoder.layers.0.norm',
    'decoder.model.3.convtr.norm': 'decoder.layers.3.norm',
    'decoder.model.4.block.1.conv.norm': 'decoder.layers.4.block.1.norm',
    'decoder.model.4.block.3.conv.norm': 'decoder.layers.4.block.3.norm',
    'decoder.model.4.shortcut.conv.norm': 'decoder.layers.4.shortcut.norm',
    'decoder.model.6.convtr.norm': 'decoder.layers.6.norm',
    'decoder.model.7.block.1.conv.norm': 'decoder.layers.7.block.1.norm',
    'decoder.model.7.block.3.conv.norm': 'decoder.layers.7.block.3.norm',
    'decoder.model.7.shortcut.conv.norm': 'decoder.layers.7.shortcut.norm',
    'decoder.model.9.convtr.norm': 'decoder.layers.9.norm',
    'decoder.model.10.block.1.conv.norm': 'decoder.layers.10.block.1.norm',
    'decoder.model.10.block.3.conv.norm': 'decoder.layers.10.block.3.norm',
    'decoder.model.10.shortcut.conv.norm': 'decoder.layers.10.shortcut.norm',
    'decoder.model.12.convtr.norm': 'decoder.layers.12.norm',
    'decoder.model.13.block.1.conv.norm': 'decoder.layers.13.block.1.norm',
    'decoder.model.13.block.3.conv.norm': 'decoder.layers.13.block.3.norm',
    'decoder.model.13.shortcut.conv.norm': 'decoder.layers.13.shortcut.norm',
    'decoder.model.15.conv.norm': 'decoder.layers.15.norm',
}
__a							     =					{
    **MAPPING_QUANTIZER,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
}
__a							     =					{
    **MAPPING_QUANTIZER,
    **MAPPING_ENCODER,
    **MAPPING_ENCODER_48K,
    **MAPPING_DECODER,
    **MAPPING_DECODER_48K,
}
__a							     =					[]
__a							     =					[]







def lowerCamelCase__  (     _lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		_lowercase ):

     '''simple docstring'''


     for attribute in key.split('''.''' ):
          UpperCAmelCase_     :				Optional[Any]  =     getattr(_lowercase					,		_lowercase )

     if weight_type is not None:
          UpperCAmelCase_     :				List[Any]  =     getattr(_lowercase					,		_lowercase ).shape
     else:
          UpperCAmelCase_     :				Optional[int]  =     hf_pointer.shape

     if hf_shape != value.shape:
          raise ValueError(
              f'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
              f''' {value.shape} for {full_name}''' )

     if weight_type == "weight":
          UpperCAmelCase_     :				Tuple  =     value
     elif weight_type == "weight_g":
          UpperCAmelCase_     :				Any  =     value
     elif weight_type == "weight_v":
          UpperCAmelCase_     :				List[Any]  =     value
     elif weight_type == "bias":
          UpperCAmelCase_     :				str  =     value
     elif weight_type == "running_mean":
          UpperCAmelCase_     :				Union[str, Any]  =     value
     elif weight_type == "running_var":
          UpperCAmelCase_     :				str  =     value
     elif weight_type == "num_batches_tracked":
          UpperCAmelCase_     :				List[Any]  =     value
     elif weight_type == "weight_ih_l0":
          UpperCAmelCase_     :				str  =     value
     elif weight_type == "weight_hh_l0":
          UpperCAmelCase_     :				str  =     value
     elif weight_type == "bias_ih_l0":
          UpperCAmelCase_     :				Optional[Any]  =     value
     elif weight_type == "bias_hh_l0":
          UpperCAmelCase_     :				int  =     value
     elif weight_type == "weight_ih_l1":
          UpperCAmelCase_     :				List[Any]  =     value
     elif weight_type == "weight_hh_l1":
          UpperCAmelCase_     :				int  =     value
     elif weight_type == "bias_ih_l1":
          UpperCAmelCase_     :				int  =     value
     elif weight_type == "bias_hh_l1":
          UpperCAmelCase_     :				List[str]  =     value
     else:
          UpperCAmelCase_     :				int  =     value

     logger.info(f'''{key + ("." + weight_type if weight_type is not None else "")} was initialized from {full_name}.''' )







def lowerCamelCase__  (     _lowercase					,		_lowercase ):

     '''simple docstring'''


     for key in ignore_keys:
          if key.endswith('''.*''' ):
               if name.startswith(key[:-1] ):
                    return True
          elif ".*." in key:
               UpperCAmelCase_,       UpperCAmelCase_     :				Dict  =     key.split('''.*.''' )
               if prefix in name and suffix in name:
                    return True
          elif key in name:
               return True
     return False







def lowerCamelCase__  (     _lowercase					,		_lowercase					,		_lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				Tuple  =     []

     if model_name == "encodec_24khz" or "encodec_32khz":
          UpperCAmelCase_     :				Tuple  =     MAPPING_24K
     elif model_name == "encodec_48khz":
          UpperCAmelCase_     :				str  =     MAPPING_48K
     else:
          raise ValueError(f'''Unsupported model: {model_name}''' )

     for name, value in orig_dict.items():
          if should_ignore(_lowercase					,		_lowercase ):
               logger.info(f'''{name} was ignored''' )
               continue

          UpperCAmelCase_     :				int  =     False
          for key, mapped_key in MAPPING.items():
               if "*" in key:
                    UpperCAmelCase_,       UpperCAmelCase_     :				Optional[int]  =     key.split('''.*.''' )
                    if prefix in name and suffix in name:
                         UpperCAmelCase_     :				Optional[Any]  =     suffix

               if key in name:
                    # HACK otherwise .embed gets initialized with .embed_avg too
                    if key.endswith('''embed''' ) and name.endswith('''embed_avg''' ):
                         continue

                    UpperCAmelCase_     :				int  =     True
                    if "*" in mapped_key:
                         UpperCAmelCase_     :				Optional[int]  =     name.split(_lowercase )[0].split('''.''' )[-2]
                         UpperCAmelCase_     :				Optional[int]  =     mapped_key.replace('''*'''					,		_lowercase )
                    if "weight_g" in name:
                         UpperCAmelCase_     :				List[Any]  =     '''weight_g'''
                    elif "weight_v" in name:
                         UpperCAmelCase_     :				Tuple  =     '''weight_v'''
                    elif "weight_ih_l0" in name:
                         UpperCAmelCase_     :				Optional[int]  =     '''weight_ih_l0'''
                    elif "weight_hh_l0" in name:
                         UpperCAmelCase_     :				Optional[int]  =     '''weight_hh_l0'''
                    elif "bias_ih_l0" in name:
                         UpperCAmelCase_     :				Dict  =     '''bias_ih_l0'''
                    elif "bias_hh_l0" in name:
                         UpperCAmelCase_     :				Any  =     '''bias_hh_l0'''
                    elif "weight_ih_l1" in name:
                         UpperCAmelCase_     :				int  =     '''weight_ih_l1'''
                    elif "weight_hh_l1" in name:
                         UpperCAmelCase_     :				int  =     '''weight_hh_l1'''
                    elif "bias_ih_l1" in name:
                         UpperCAmelCase_     :				Dict  =     '''bias_ih_l1'''
                    elif "bias_hh_l1" in name:
                         UpperCAmelCase_     :				Union[str, Any]  =     '''bias_hh_l1'''
                    elif "bias" in name:
                         UpperCAmelCase_     :				List[str]  =     '''bias'''
                    elif "weight" in name:
                         UpperCAmelCase_     :				int  =     '''weight'''
                    elif "running_mean" in name:
                         UpperCAmelCase_     :				Tuple  =     '''running_mean'''
                    elif "running_var" in name:
                         UpperCAmelCase_     :				Dict  =     '''running_var'''
                    elif "num_batches_tracked" in name:
                         UpperCAmelCase_     :				Tuple  =     '''num_batches_tracked'''
                    else:
                         UpperCAmelCase_     :				Dict  =     None
                    set_recursively(_lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		_lowercase )
               continue
          if not is_used:
               unused_weights.append(_lowercase )

     logger.warning(f'''Unused weights: {unused_weights}''' )







@torch.no_grad()
def lowerCamelCase__  (     _lowercase					,		_lowercase					,		_lowercase					,		_lowercase=None					,		_lowercase=None					,		):

     '''simple docstring'''


     if config_path is not None:
          UpperCAmelCase_     :				int  =     EncodecConfig.from_pretrained(_lowercase )
     else:
          UpperCAmelCase_     :				List[str]  =     EncodecConfig()

     if model_name == "encodec_24khz":
          pass  # config is already correct
     elif model_name == "encodec_32khz":
          UpperCAmelCase_     :				Union[str, Any]  =     [8, 5, 4, 4]
          UpperCAmelCase_     :				str  =     [2.2]
          UpperCAmelCase_     :				Optional[Any]  =     64
          UpperCAmelCase_     :				str  =     32000
          UpperCAmelCase_     :				Tuple  =     2048
          UpperCAmelCase_     :				str  =     False
          UpperCAmelCase_     :				List[Any]  =     False
          UpperCAmelCase_     :				int  =     False
     elif model_name == "encodec_48khz":
          UpperCAmelCase_     :				List[str]  =     [8, 5, 4, 2]
          UpperCAmelCase_     :				Optional[int]  =     [3.0, 6.0, 12.0, 24.0]
          UpperCAmelCase_     :				Optional[int]  =     48000
          UpperCAmelCase_     :				List[str]  =     2
          UpperCAmelCase_     :				Optional[Any]  =     False
          UpperCAmelCase_     :				Any  =     '''time_group_norm'''
          UpperCAmelCase_     :				Union[str, Any]  =     True
          UpperCAmelCase_     :				Union[str, Any]  =     1.0
          UpperCAmelCase_     :				List[Any]  =     0.01
     else:
          raise ValueError(f'''Unknown model name: {model_name}''' )

     UpperCAmelCase_     :				int  =     EncodecModel(_lowercase )

     UpperCAmelCase_     :				List[Any]  =     EncodecFeatureExtractor(
         feature_size=config.audio_channels					,		sampling_rate=config.sampling_rate					,		chunk_length_s=config.chunk_length_s					,		overlap=config.overlap					,		)
     feature_extractor.save_pretrained(_lowercase )

     UpperCAmelCase_     :				int  =     torch.load(_lowercase )
     if "best_state" in original_checkpoint:
          # we might have a training state saved, in which case discard the yaml results and just retain the weights
          UpperCAmelCase_     :				Optional[int]  =     original_checkpoint['''best_state''']
     recursively_load_weights(_lowercase					,		_lowercase					,		_lowercase )
     model.save_pretrained(_lowercase )

     if repo_id:
          print('''Pushing to the hub...''' )
          feature_extractor.push_to_hub(_lowercase )
          model.push_to_hub(_lowercase )


if __name__ == "__main__":
     __a							     =					argparse.ArgumentParser()
     parser.add_argument(
         '--model',
         default='encodec_24khz',
         type=str,
         help='The model to convert. Should be one of \'encodec_24khz\', \'encodec_32khz\', \'encodec_48khz\'.',
     )
     parser.add_argument('--checkpoint_path', required=True, default=None, type=str, help='Path to original checkpoint')
     parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
     parser.add_argument(
         '--pytorch_dump_folder_path', required=True, default=None, type=str, help='Path to the output PyTorch model.'
     )
     parser.add_argument(
         '--push_to_hub', default=None, type=str, help='Where to upload the converted model on the 🤗 hub.'
     )

     __a							     =					parser.parse_args()
     convert_checkpoint(
         args.model,
         args.checkpoint_path,
         args.pytorch_dump_folder_path,
         args.config_path,
         args.push_to_hub,
     )





import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
     import PIL.Image

     from .features import FeatureType


__a							     =					None
__a							     =					'<' if sys.byteorder == 'little' else '>'
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
__a							     =					[
    np.dtype('|b1'),
    np.dtype('|u1'),
    np.dtype('<u2'),
    np.dtype('>u2'),
    np.dtype('<i2'),
    np.dtype('>i2'),
    np.dtype('<u4'),
    np.dtype('>u4'),
    np.dtype('<i4'),
    np.dtype('>i4'),
    np.dtype('<f4'),
    np.dtype('>f4'),
    np.dtype('<f8'),
    np.dtype('>f8'),
]

@dataclass
class     __a:






    """simple docstring"""





    lowerCAmelCase         = True
    lowerCAmelCase         = None
    # Automatically constructed
    lowerCAmelCase         = "PIL.Image.Image"
    lowerCAmelCase         = pa.struct({'''bytes''': pa.binary(), '''path''': pa.string()}  )
    lowerCAmelCase         = field(default='''Image'''							,		init=_a							,		repr=_a  )





    def __call__( self    )					->						Tuple:
         return self.pa_type





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						dict:
         if config.PIL_AVAILABLE:
              import PIL.Image
         else:
              raise ImportError('''To support encoding images, please install \'Pillow\'.'''    )

         if isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              UpperCAmelCase_     :				List[str]  =     np.array(_SCREAMING_SNAKE_CASE    )

         if isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              return {"path": value, "bytes": None}
         elif isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              return {"path": None, "bytes": value}
         elif isinstance(_SCREAMING_SNAKE_CASE  ,np.ndarray    ):
              # convert the image array to PNG/TIFF bytes
              return encode_np_array(_SCREAMING_SNAKE_CASE    )
         elif isinstance(_SCREAMING_SNAKE_CASE  ,PIL.Image.Image    ):
              # convert the PIL image to bytes (default format is PNG/TIFF)
              return encode_pil_image(_SCREAMING_SNAKE_CASE    )
         elif value.get('''path'''    ) is not None and os.path.isfile(value['''path''']    ):
              # we set "bytes": None to not duplicate the data if they're already available locally
              return {"bytes": None, "path": value.get('''path'''    )}
         elif value.get('''bytes'''    ) is not None or value.get('''path'''    ) is not None:
              # store the image bytes, and path is used to infer the image format using the file extension
              return {"bytes": value.get('''bytes'''    ), "path": value.get('''path'''    )}
         else:
              raise ValueError(
                  f'''An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''    )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE=None    )					->						"PIL.Image.Image":
         if not self.decode:
              raise RuntimeError('''Decoding is disabled for this feature. Please use Image(decode=True) instead.'''    )

         if config.PIL_AVAILABLE:
              import PIL.Image
         else:
              raise ImportError('''To support decoding images, please install \'Pillow\'.'''    )

         if token_per_repo_id is None:
              UpperCAmelCase_     :				Dict  =     {}

         UpperCAmelCase_,       UpperCAmelCase_     :				Union[str, Any]  =     value['''path'''], value['''bytes''']
         if bytes_ is None:
              if path is None:
                   raise ValueError(f'''An image should have one of \'path\' or \'bytes\' but both are None in {value}.'''    )
              else:
                   if is_local_path(_SCREAMING_SNAKE_CASE    ):
                        UpperCAmelCase_     :				Tuple  =     PIL.Image.open(_SCREAMING_SNAKE_CASE    )
                   else:
                        UpperCAmelCase_     :				Dict  =     path.split('''::'''    )[-1]
                        try:
                             UpperCAmelCase_     :				Optional[int]  =     string_to_dict(_SCREAMING_SNAKE_CASE  ,config.HUB_DATASETS_URL    )['''repo_id''']
                             UpperCAmelCase_     :				Tuple  =     token_per_repo_id.get(_SCREAMING_SNAKE_CASE    )
                        except ValueError:
                             UpperCAmelCase_     :				Optional[Any]  =     None
                        with xopen(_SCREAMING_SNAKE_CASE  ,'''rb'''  ,use_auth_token=_SCREAMING_SNAKE_CASE    ) as f:
                             UpperCAmelCase_     :				List[str]  =     BytesIO(f.read()    )
                        UpperCAmelCase_     :				Optional[Any]  =     PIL.Image.open(bytes_    )
         else:
              UpperCAmelCase_     :				List[Any]  =     PIL.Image.open(BytesIO(bytes_    )    )
         image.load()  # to avoid "Too many open files" errors
         return image





    def 				a__ ( self    )					->						Union["FeatureType", Dict[str, "FeatureType"]]:
         from .features import Value

         return (
             self
             if self.decode
             else {
                 "bytes": Value('''binary'''    ),
                 "path": Value('''string'''    ),
             }
         )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						pa.StructArray:
         if pa.types.is_string(storage.type    ):
              UpperCAmelCase_     :				Dict  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.binary()    )
              UpperCAmelCase_     :				Dict  =     pa.StructArray.from_arrays([bytes_array, storage]  ,['''bytes''', '''path''']  ,mask=storage.is_null()    )
         elif pa.types.is_binary(storage.type    ):
              UpperCAmelCase_     :				List[str]  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.string()    )
              UpperCAmelCase_     :				Tuple  =     pa.StructArray.from_arrays([storage, path_array]  ,['''bytes''', '''path''']  ,mask=storage.is_null()    )
         elif pa.types.is_struct(storage.type    ):
              if storage.type.get_field_index('''bytes'''    ) >= 0:
                   UpperCAmelCase_     :				Dict  =     storage.field('''bytes'''    )
              else:
                   UpperCAmelCase_     :				Any  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.binary()    )
              if storage.type.get_field_index('''path'''    ) >= 0:
                   UpperCAmelCase_     :				int  =     storage.field('''path'''    )
              else:
                   UpperCAmelCase_     :				List[str]  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.string()    )
              UpperCAmelCase_     :				Optional[Any]  =     pa.StructArray.from_arrays([bytes_array, path_array]  ,['''bytes''', '''path''']  ,mask=storage.is_null()    )
         elif pa.types.is_list(storage.type    ):
              UpperCAmelCase_     :				Optional[Any]  =     pa.array(
                  [encode_np_array(np.array(_SCREAMING_SNAKE_CASE    )    )['''bytes'''] if arr is not None else None for arr in storage.to_pylist()]  ,type=pa.binary()  ,)
              UpperCAmelCase_     :				Any  =     pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,type=pa.string()    )
              UpperCAmelCase_     :				Dict  =     pa.StructArray.from_arrays(
                  [bytes_array, path_array]  ,['''bytes''', '''path''']  ,mask=bytes_array.is_null()    )
         return array_cast(_SCREAMING_SNAKE_CASE  ,self.pa_type    )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						pa.StructArray:

         @no_op_if_value_is_null
         def path_to_bytes(_SCREAMING_SNAKE_CASE    ):
              with xopen(_SCREAMING_SNAKE_CASE  ,'''rb'''    ) as f:
                   UpperCAmelCase_     :				Any  =     f.read()
              return bytes_

         UpperCAmelCase_     :				Union[str, Any]  =     pa.array(
             [
                 (path_to_bytes(x['''path''']    ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
                 for x in storage.to_pylist()
             ]  ,type=pa.binary()  ,)
         UpperCAmelCase_     :				List[str]  =     pa.array(
             [os.path.basename(_SCREAMING_SNAKE_CASE    ) if path is not None else None for path in storage.field('''path'''    ).to_pylist()]  ,type=pa.string()  ,)
         UpperCAmelCase_     :				Union[str, Any]  =     pa.StructArray.from_arrays([bytes_array, path_array]  ,['''bytes''', '''path''']  ,mask=bytes_array.is_null()    )
         return array_cast(_SCREAMING_SNAKE_CASE  ,self.pa_type    )







def lowerCamelCase__  (     ):

     '''simple docstring'''


     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     global _IMAGE_COMPRESSION_FORMATS
     if _IMAGE_COMPRESSION_FORMATS is None:
          PIL.Image.init()
          UpperCAmelCase_     :				Optional[int]  =     list(set(PIL.Image.OPEN.keys() ) & set(PIL.Image.SAVE.keys() ) )
     return _IMAGE_COMPRESSION_FORMATS







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				Optional[int]  =     BytesIO()
     if image.format in list_image_compression_formats():
          UpperCAmelCase_     :				int  =     image.format
     else:
          UpperCAmelCase_     :				List[Any]  =     '''PNG''' if image.mode in ['''1''', '''L''', '''LA''', '''RGB''', '''RGBA'''] else '''TIFF'''
     image.save(_lowercase					,		format=_lowercase )
     return buffer.getvalue()







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if hasattr(_lowercase					,		'''filename''' ) and image.filename != "":
          return {"path": image.filename, "bytes": None}
     else:
          return {"path": None, "bytes": image_to_bytes(_lowercase )}







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     UpperCAmelCase_     :				Tuple  =     array.dtype
     UpperCAmelCase_     :				List[str]  =     dtype.byteorder if dtype.byteorder != '''=''' else _NATIVE_BYTEORDER
     UpperCAmelCase_     :				Dict  =     dtype.kind
     UpperCAmelCase_     :				Union[str, Any]  =     dtype.itemsize

     UpperCAmelCase_     :				Optional[Any]  =     None

     # Multi-channel array case (only np.dtype("|u1") is allowed)
     if array.shape[2:]:
          UpperCAmelCase_     :				Tuple  =     np.dtype('''|u1''' )
          if dtype_kind not in ["u", "i"]:
               raise TypeError(
                   f'''Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.''' )
          if dtype is not dest_dtype:
               warnings.warn(f'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\'''' )
    # Exact match
     elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
          UpperCAmelCase_     :				Union[str, Any]  =     dtype
     else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
          while dtype_itemsize >= 1:
               UpperCAmelCase_     :				Union[str, Any]  =     dtype_byteorder + dtype_kind + str(_lowercase )
               UpperCAmelCase_     :				str  =     np.dtype(_lowercase )
               if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
                    warnings.warn(f'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\'''' )
                    break
               else:
                    dtype_itemsize //= 2

     if dest_dtype is None:
          raise TypeError(
              f'''Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}''' )
     UpperCAmelCase_     :				Any  =     PIL.Image.fromarray(array.astype(_lowercase ) )
     return {"path": None, "bytes": image_to_bytes(_lowercase )}







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if config.PIL_AVAILABLE:
          import PIL.Image
     else:
          raise ImportError('''To support encoding images, please install \'Pillow\'.''' )

     if objs:
          UpperCAmelCase_,       UpperCAmelCase_     :				Tuple  =     first_non_null_value(_lowercase )
          if isinstance(_lowercase					,		_lowercase ):
               return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
          if isinstance(_lowercase					,		np.ndarray ):
               UpperCAmelCase_     :				Any  =     no_op_if_value_is_null(_lowercase )
               return [obj_to_image_dict_func(_lowercase ) for obj in objs]
          elif isinstance(_lowercase					,		PIL.Image.Image ):
               UpperCAmelCase_     :				Union[str, Any]  =     no_op_if_value_is_null(_lowercase )
               return [obj_to_image_dict_func(_lowercase ) for obj in objs]
          else:
               return objs
     else:
          return objs