


'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase												=						logging.get_logger(__name__)


__lowerCamelCase												=						{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

__lowerCamelCase												=						{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

__lowerCamelCase												=						{
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def        a__     (					):
   UpperCAmelCase__	:Optional[int]      =      (
       list(range(ord('''!'''						),   ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						),   ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						),   ord('''ÿ'''						) + 1						)						)
   )
   UpperCAmelCase__	:List[Any]      =      bs[:]
   UpperCAmelCase__	:Dict      =      0
   for b in range(2**8						):
      if b not in bs:
         bs.append(UpperCamelCase_						)
         cs.append(2**8 + n						)
         n += 1
   UpperCAmelCase__	:Any      =      [chr(UpperCamelCase_						) for n in cs]
   return dict(zip(UpperCamelCase_,   UpperCamelCase_						)						)


def        a__     (					UpperCamelCase_						:      str						):
   UpperCAmelCase__	:Optional[int]      =      set()
   UpperCAmelCase__	:Any      =      word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)						)
      UpperCAmelCase__	:List[Any]      =      char
   return pairs




class 				UpperCAmelCase			(			_snake_case						):
 UpperCAmelCase				=					VOCAB_FILES_NAMES
 UpperCAmelCase				=					PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase				=					["input_ids", "attention_mask"]


 def __init__(      self							:	Optional[int]			,       __lowerCamelCase							:	Optional[Any]			,       __lowerCamelCase							:	Dict			,       __lowerCamelCase							:	Tuple="replace"			,       __lowerCamelCase							:	Optional[int]="<s>"			,       __lowerCamelCase							:	Tuple="</s>"			,       __lowerCamelCase							:	List[str]="</s>"			,       __lowerCamelCase							:	Optional[int]="<s>"			,       __lowerCamelCase							:	Dict="<unk>"			,       __lowerCamelCase							:	Union[str, Any]="<pad>"			,       __lowerCamelCase							:	Optional[int]="<mask>"			,       __lowerCamelCase							:	Any=False			,       **__lowerCamelCase							:	Any			,       ):
    UpperCAmelCase__	:str      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else bos_token
    UpperCAmelCase__	:Optional[int]      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else eos_token
    UpperCAmelCase__	:int      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else sep_token
    UpperCAmelCase__	:Optional[int]      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else cls_token
    UpperCAmelCase__	:List[Any]      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else unk_token
    UpperCAmelCase__	:str      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    UpperCAmelCase__	:Any      =      AddedToken(__lowerCamelCase			,       lstrip=__lowerCamelCase			,       rstrip=__lowerCamelCase ) if isinstance(__lowerCamelCase			,       __lowerCamelCase ) else mask_token

    super().__init__(
        errors=__lowerCamelCase			,       bos_token=__lowerCamelCase			,       eos_token=__lowerCamelCase			,       unk_token=__lowerCamelCase			,       sep_token=__lowerCamelCase			,       cls_token=__lowerCamelCase			,       pad_token=__lowerCamelCase			,       mask_token=__lowerCamelCase			,       add_prefix_space=__lowerCamelCase			,       **__lowerCamelCase			,       )

    with open(__lowerCamelCase			,       encoding='''utf-8''' ) as vocab_handle:
       UpperCAmelCase__	:Union[str, Any]      =      json.load(__lowerCamelCase )
    UpperCAmelCase__	:List[Any]      =      {v: k for k, v in self.encoder.items()}
    UpperCAmelCase__	:Tuple      =      errors  # how to handle errors in decoding
    UpperCAmelCase__	:Tuple      =      bytes_to_unicode()
    UpperCAmelCase__	:Optional[int]      =      {v: k for k, v in self.byte_encoder.items()}
    with open(__lowerCamelCase			,       encoding='''utf-8''' ) as merges_handle:
       UpperCAmelCase__	:Union[str, Any]      =      merges_handle.read().split('''\n''' )[1:-1]
    UpperCAmelCase__	:Optional[int]      =      [tuple(merge.split() ) for merge in bpe_merges]
    UpperCAmelCase__	:Tuple      =      dict(zip(__lowerCamelCase			,       range(len(__lowerCamelCase ) ) ) )
    UpperCAmelCase__	:Union[str, Any]      =      {}
    UpperCAmelCase__	:Optional[Any]      =      add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    UpperCAmelCase__	:Union[str, Any]      =      re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''' )


 @property
 def        __SCREAMING_SNAKE_CASE					(      self							:	Dict ):
    return len(self.encoder )


 def        __SCREAMING_SNAKE_CASE					(      self							:	str ):
    return dict(self.encoder			,       **self.added_tokens_encoder )


 def        __SCREAMING_SNAKE_CASE					(      self							:	Any			,       __lowerCamelCase							:	int ):
    if token in self.cache:
       return self.cache[token]
    UpperCAmelCase__	:Optional[int]      =      tuple(__lowerCamelCase )
    UpperCAmelCase__	:Optional[Any]      =      get_pairs(__lowerCamelCase )

    if not pairs:
       return token

    while True:
       UpperCAmelCase__	:str      =      min(__lowerCamelCase			,       key=lambda __lowerCamelCase : self.bpe_ranks.get(__lowerCamelCase			,       float('''inf''' ) ) )
       if bigram not in self.bpe_ranks:
          break
       UpperCAmelCase__   ,						UpperCAmelCase__	:str      =      bigram
       UpperCAmelCase__	:Optional[Any]      =      []
       UpperCAmelCase__	:int      =      0
       while i < len(__lowerCamelCase ):
          try:
             UpperCAmelCase__	:List[Any]      =      word.index(__lowerCamelCase			,       __lowerCamelCase )
          except ValueError:
             new_word.extend(word[i:] )
             break
          else:
             new_word.extend(word[i:j] )
             UpperCAmelCase__	:str      =      j

          if word[i] == first and i < len(__lowerCamelCase ) - 1 and word[i + 1] == second:
             new_word.append(first + second )
             i += 2
          else:
             new_word.append(word[i] )
             i += 1
       UpperCAmelCase__	:Optional[Any]      =      tuple(__lowerCamelCase )
       UpperCAmelCase__	:List[Any]      =      new_word
       if len(__lowerCamelCase ) == 1:
          break
       else:
          UpperCAmelCase__	:List[str]      =      get_pairs(__lowerCamelCase )
    UpperCAmelCase__	:Optional[Any]      =      ''' '''.join(__lowerCamelCase )
    UpperCAmelCase__	:Optional[int]      =      word
    return word


 def        __SCREAMING_SNAKE_CASE					(      self							:	Union[str, Any]			,       __lowerCamelCase							:	Dict ):
    UpperCAmelCase__	:Any      =      []
    for token in re.findall(self.pat			,       __lowerCamelCase ):
       UpperCAmelCase__	:str      =      ''''''.join(
           self.byte_encoder[b] for b in token.encode('''utf-8''' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase ).split(''' ''' ) )
    return bpe_tokens


 def        __SCREAMING_SNAKE_CASE					(      self							:	str			,       __lowerCamelCase							:	List[str] ):
    return self.encoder.get(__lowerCamelCase			,       self.encoder.get(self.unk_token ) )


 def        __SCREAMING_SNAKE_CASE					(      self							:	Tuple			,       __lowerCamelCase							:	List[str] ):
    return self.decoder.get(__lowerCamelCase )


 def        __SCREAMING_SNAKE_CASE					(      self							:	int			,       __lowerCamelCase							:	Optional[Any] ):
    UpperCAmelCase__	:List[Any]      =      ''''''.join(__lowerCamelCase )
    UpperCAmelCase__	:List[Any]      =      bytearray([self.byte_decoder[c] for c in text] ).decode('''utf-8'''			,       errors=self.errors )
    return text


 def        __SCREAMING_SNAKE_CASE					(      self							:	List[str]			,       __lowerCamelCase							:	str			,       __lowerCamelCase							:	Optional[str] = None ):
    if not os.path.isdir(__lowerCamelCase ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
       return
    UpperCAmelCase__	:Any      =      os.path.join(
        __lowerCamelCase			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
    UpperCAmelCase__	:Any      =      os.path.join(
        __lowerCamelCase			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

    with open(__lowerCamelCase			,       '''w'''			,       encoding='''utf-8''' ) as f:
       f.write(json.dumps(self.encoder			,       indent=2			,       sort_keys=__lowerCamelCase			,       ensure_ascii=__lowerCamelCase ) + '''\n''' )

    UpperCAmelCase__	:int      =      0
    with open(__lowerCamelCase			,       '''w'''			,       encoding='''utf-8''' ) as writer:
       writer.write('''#version: 0.2\n''' )
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,       key=lambda __lowerCamelCase : kv[1] ):
          if index != token_index:
             logger.warning(
                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                 ''' Please check that the tokenizer is not corrupted!''' )
             UpperCAmelCase__	:Union[str, Any]      =      token_index
          writer.write(''' '''.join(__lowerCamelCase ) + '''\n''' )
          index += 1

    return vocab_file, merge_file


 def        __SCREAMING_SNAKE_CASE					(      self							:	Any			,       __lowerCamelCase							:	List[int]			,       __lowerCamelCase							:	Optional[List[int]] = None ):
    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    UpperCAmelCase__	:Dict      =      [self.cls_token_id]
    UpperCAmelCase__	:int      =      [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep


 def        __SCREAMING_SNAKE_CASE					(      self							:	int			,       __lowerCamelCase							:	List[int]			,       __lowerCamelCase							:	Optional[List[int]] = None			,       __lowerCamelCase							:	bool = False ):
    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase			,       token_ids_a=__lowerCamelCase			,       already_has_special_tokens=__lowerCamelCase )

    if token_ids_a is None:
       return [1] + ([0] * len(__lowerCamelCase )) + [1]
    return [1] + ([0] * len(__lowerCamelCase )) + [1, 1] + ([0] * len(__lowerCamelCase )) + [1]


 def        __SCREAMING_SNAKE_CASE					(      self							:	Dict			,       __lowerCamelCase							:	List[int]			,       __lowerCamelCase							:	Optional[List[int]] = None ):
    UpperCAmelCase__	:Optional[Any]      =      [self.sep_token_id]
    UpperCAmelCase__	:Tuple      =      [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]





 def        __SCREAMING_SNAKE_CASE					(      self							:	Union[str, Any]			,       __lowerCamelCase							:	Union[str, Any]			,       __lowerCamelCase							:	Tuple=False			,       **__lowerCamelCase							:	List[Any] ):
    UpperCAmelCase__	:str      =      kwargs.pop('''add_prefix_space'''			,       self.add_prefix_space )
    if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase ) > 0 and not text[0].isspace()):
       UpperCAmelCase__	:Optional[int]      =      ''' ''' + text
    return (text, kwargs)







'''simple docstring'''





import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
    from .features import FeatureType


__lowerCamelCase					,		__lowerCamelCase					,		__lowerCamelCase												=						False, False, False




@dataclass
class 				UpperCAmelCase			:
 UpperCAmelCase				=					None
 UpperCAmelCase				=					True
 UpperCAmelCase				=					True
 UpperCAmelCase				=					None
 # Automatically constructed
 UpperCAmelCase				=					"dict"
 UpperCAmelCase				=					pa.struct({"bytes": pa.binary(), "path": pa.string()}						)
 UpperCAmelCase				=					field(default="Audio"						, init=_snake_case						, repr=_snake_case						)


 def __call__(      self							:	Any ):
    return self.pa_type


 def        __SCREAMING_SNAKE_CASE					(      self							:	Dict			,       __lowerCamelCase							:	Union[str, bytes, dict] ):
    try:
       import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
    except ImportError as err:
       raise ImportError('''To support encoding audio data, please install \'soundfile\'.''' ) from err
    if isinstance(__lowerCamelCase			,       __lowerCamelCase ):
       return {"bytes": None, "path": value}
    elif isinstance(__lowerCamelCase			,       __lowerCamelCase ):
       return {"bytes": value, "path": None}
    elif "array" in value:
       # convert the audio array to wav bytes
       UpperCAmelCase__	:int      =      BytesIO()
       sf.write(__lowerCamelCase			,       value['''array''']			,       value['''sampling_rate''']			,       format='''wav''' )
       return {"bytes": buffer.getvalue(), "path": None}
    elif value.get('''path''' ) is not None and os.path.isfile(value['''path'''] ):
       # we set "bytes": None to not duplicate the data if they're already available locally
       if value["path"].endswith('''pcm''' ):
          # "PCM" only has raw audio bytes
          if value.get('''sampling_rate''' ) is None:
             # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
             raise KeyError('''To use PCM files, please specify a \'sampling_rate\' in Audio object''' )
          if value.get('''bytes''' ):
             # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
             UpperCAmelCase__	:List[Any]      =      np.frombuffer(value['''bytes''']			,       dtype=np.intaa ).astype(np.floataa ) / 3_2_7_6_7
          else:
             UpperCAmelCase__	:Optional[Any]      =      np.memmap(value['''path''']			,       dtype='''h'''			,       mode='''r''' ).astype(np.floataa ) / 3_2_7_6_7

          UpperCAmelCase__	:Optional[Any]      =      BytesIO(bytes() )
          sf.write(__lowerCamelCase			,       __lowerCamelCase			,       value['''sampling_rate''']			,       format='''wav''' )
          return {"bytes": buffer.getvalue(), "path": None}
       else:
          return {"bytes": None, "path": value.get('''path''' )}
    elif value.get('''bytes''' ) is not None or value.get('''path''' ) is not None:
       # store the audio bytes, and path is used to infer the audio format using the file extension
       return {"bytes": value.get('''bytes''' ), "path": value.get('''path''' )}
    else:
       raise ValueError(
           f'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.''' )


 def        __SCREAMING_SNAKE_CASE					(      self							:	List[Any]			,       __lowerCamelCase							:	dict			,       __lowerCamelCase							:	Optional[Dict[str, Union[str, bool, None]]] = None ):
    if not self.decode:
       raise RuntimeError('''Decoding is disabled for this feature. Please use Audio(decode=True) instead.''' )

    UpperCAmelCase__   ,						UpperCAmelCase__	:str      =      (value['''path'''], BytesIO(value['''bytes'''] )) if value['''bytes'''] is not None else (value['''path'''], None)
    if path is None and file is None:
       raise ValueError(f'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.''' )

    try:
       import librosa
       import soundfile as sf
    except ImportError as err:
       raise ImportError('''To support decoding audio files, please install \'librosa\' and \'soundfile\'.''' ) from err

    UpperCAmelCase__	:List[str]      =      xsplitext(__lowerCamelCase )[1][1:].lower() if path is not None else None
    if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
       raise RuntimeError(
           '''Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, '''
           '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ''' )
    elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
       raise RuntimeError(
           '''Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, '''
           '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ''' )

    if file is None:
       UpperCAmelCase__	:Optional[Any]      =      token_per_repo_id or {}
       UpperCAmelCase__	:str      =      path.split('''::''' )[-1]
       try:
          UpperCAmelCase__	:Tuple      =      string_to_dict(__lowerCamelCase			,       config.HUB_DATASETS_URL )['''repo_id''']
          UpperCAmelCase__	:str      =      token_per_repo_id[repo_id]
       except (ValueError, KeyError):
          UpperCAmelCase__	:Tuple      =      None

       with xopen(__lowerCamelCase			,       '''rb'''			,       use_auth_token=__lowerCamelCase ) as f:
          UpperCAmelCase__   ,						UpperCAmelCase__	:Union[str, Any]      =      sf.read(__lowerCamelCase )

    else:
       UpperCAmelCase__   ,						UpperCAmelCase__	:List[Any]      =      sf.read(__lowerCamelCase )

    UpperCAmelCase__	:Optional[int]      =      array.T
    if self.mono:
       UpperCAmelCase__	:Any      =      librosa.to_mono(__lowerCamelCase )
    if self.sampling_rate and self.sampling_rate != sampling_rate:
       UpperCAmelCase__	:Union[str, Any]      =      librosa.resample(__lowerCamelCase			,       orig_sr=__lowerCamelCase			,       target_sr=self.sampling_rate )
       UpperCAmelCase__	:List[str]      =      self.sampling_rate

    return {"path": path, "array": array, "sampling_rate": sampling_rate}


 def        __SCREAMING_SNAKE_CASE					(      self							:	Optional[Any] ):
    from .features import Value

    if self.decode:
       raise ValueError('''Cannot flatten a decoded Audio feature.''' )
    return {
        "bytes": Value('''binary''' ),
        "path": Value('''string''' ),
    }


 def        __SCREAMING_SNAKE_CASE					(      self							:	Any			,       __lowerCamelCase							:	Union[pa.StringArray, pa.StructArray] ):
    if pa.types.is_string(storage.type ):
       UpperCAmelCase__	:List[str]      =      pa.array([None] * len(__lowerCamelCase )			,       type=pa.binary() )
       UpperCAmelCase__	:Tuple      =      pa.StructArray.from_arrays([bytes_array, storage]			,       ['''bytes''', '''path''']			,       mask=storage.is_null() )
    elif pa.types.is_binary(storage.type ):
       UpperCAmelCase__	:str      =      pa.array([None] * len(__lowerCamelCase )			,       type=pa.string() )
       UpperCAmelCase__	:int      =      pa.StructArray.from_arrays([storage, path_array]			,       ['''bytes''', '''path''']			,       mask=storage.is_null() )
    elif pa.types.is_struct(storage.type ) and storage.type.get_all_field_indices('''array''' ):
       UpperCAmelCase__	:Any      =      pa.array([Audio().encode_example(__lowerCamelCase ) if x is not None else None for x in storage.to_pylist()] )
    elif pa.types.is_struct(storage.type ):
       if storage.type.get_field_index('''bytes''' ) >= 0:
          UpperCAmelCase__	:str      =      storage.field('''bytes''' )
       else:
          UpperCAmelCase__	:List[str]      =      pa.array([None] * len(__lowerCamelCase )			,       type=pa.binary() )
       if storage.type.get_field_index('''path''' ) >= 0:
          UpperCAmelCase__	:Optional[int]      =      storage.field('''path''' )
       else:
          UpperCAmelCase__	:Optional[int]      =      pa.array([None] * len(__lowerCamelCase )			,       type=pa.string() )
       UpperCAmelCase__	:List[str]      =      pa.StructArray.from_arrays([bytes_array, path_array]			,       ['''bytes''', '''path''']			,       mask=storage.is_null() )
    return array_cast(__lowerCamelCase			,       self.pa_type )





 def        __SCREAMING_SNAKE_CASE					(      self							:	Optional[Any]			,       __lowerCamelCase							:	pa.StructArray ):

    @no_op_if_value_is_null
    def path_to_bytes(__lowerCamelCase							:	Dict ):
       with xopen(__lowerCamelCase			,       '''rb''' ) as f:
          UpperCAmelCase__	:Any      =      f.read()
       return bytes_

    UpperCAmelCase__	:Union[str, Any]      =      pa.array(
        [
            (path_to_bytes(x['''path'''] ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
            for x in storage.to_pylist()
        ]			,       type=pa.binary()			,       )
    UpperCAmelCase__	:Optional[int]      =      pa.array(
        [os.path.basename(__lowerCamelCase ) if path is not None else None for path in storage.field('''path''' ).to_pylist()]			,       type=pa.string()			,       )
    UpperCAmelCase__	:Optional[int]      =      pa.StructArray.from_arrays([bytes_array, path_array]			,       ['''bytes''', '''path''']			,       mask=bytes_array.is_null() )
    return array_cast(__lowerCamelCase			,       self.pa_type )



