






import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
 from .features import FeatureType


lowerCamelCase      ,					lowerCamelCase      ,					lowerCamelCase			:					int         =	False, False, False



@dataclass
class 							_UpperCamelCase			:
      snake_case_  =							None
      snake_case_  =							True
      snake_case_  =							True
      snake_case_  =							None
      # Automatically constructed
      snake_case_  =							"dict"
      snake_case_  =							pa.struct({"""bytes""": pa.binary(), """path""": pa.string()}			)
      snake_case_  =							field(default="""Audio"""							,		init=a_							,		repr=a_			)


      def __call__(	self	)->	str:
           return self.pa_type


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	dict:
           try:
                import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
           except ImportError as err:
                raise ImportError("To support encoding audio data, please install 'soundfile'."	) from err
           if isinstance(__UpperCamelCase					,						__UpperCamelCase	):
                return {"bytes": None, "path": value}
           elif isinstance(__UpperCamelCase					,						__UpperCamelCase	):
                return {"bytes": value, "path": None}
           elif "array" in value:
                # convert the audio array to wav bytes
                __lowerCAmelCase								=					BytesIO()
                sf.write(__UpperCamelCase					,						value["array"]					,						value["sampling_rate"]					,						format="wav"	)
                return {"bytes": buffer.getvalue(), "path": None}
           elif value.get("path"	) is not None and os.path.isfile(value["path"]	):
                # we set "bytes": None to not duplicate the data if they're already available locally
                if value["path"].endswith("pcm"	):
                     # "PCM" only has raw audio bytes
                     if value.get("sampling_rate"	) is None:
                          # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
                          raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object"	)
                     if value.get("bytes"	):
                          # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
                          __lowerCAmelCase								=					np.frombuffer(value["bytes"]					,						dtype=np.intaa	).astype(np.floataa	) / 3_2_7_6_7
                     else:
                          __lowerCAmelCase								=					np.memmap(value["path"]					,						dtype="h"					,						mode="r"	).astype(np.floataa	) / 3_2_7_6_7

                     __lowerCAmelCase								=					BytesIO(bytes()	)
                     sf.write(__UpperCamelCase					,						__UpperCamelCase					,						value["sampling_rate"]					,						format="wav"	)
                     return {"bytes": buffer.getvalue(), "path": None}
                else:
                     return {"bytes": None, "path": value.get("path"	)}
           elif value.get("bytes"	) is not None or value.get("path"	) is not None:
                # store the audio bytes, and path is used to infer the audio format using the file extension
                return {"bytes": value.get("bytes"	), "path": value.get("path"	)}
           else:
                raise ValueError(
                    F"""An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}."""	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase = None	)->	dict:
           if not self.decode:
                raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead."	)

           __lowerCAmelCase			, __lowerCAmelCase								=					(value["path"], BytesIO(value["bytes"]	)) if value["bytes"] is not None else (value["path"], None)
           if path is None and file is None:
                raise ValueError(F"""An audio sample should have one of 'path' or 'bytes' but both are None in {value}."""	)

           try:
                import librosa
                import soundfile as sf
           except ImportError as err:
                raise ImportError("To support decoding audio files, please install 'librosa' and 'soundfile'."	) from err

           __lowerCAmelCase								=					xsplitext(__UpperCamelCase	)[1][1:].lower() if path is not None else None
           if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
                raise RuntimeError(
                    "Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, "
                    "You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. "	)
           elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
                raise RuntimeError(
                    "Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, "
                    "You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. "	)

           if file is None:
                __lowerCAmelCase								=					token_per_repo_id or {}
                __lowerCAmelCase								=					path.split("::"	)[-1]
                try:
                     __lowerCAmelCase								=					string_to_dict(__UpperCamelCase					,						config.HUB_DATASETS_URL	)["repo_id"]
                     __lowerCAmelCase								=					token_per_repo_id[repo_id]
                except (ValueError, KeyError):
                     __lowerCAmelCase								=					None

                with xopen(__UpperCamelCase					,						"rb"					,						use_auth_token=__UpperCamelCase	) as f:
                     __lowerCAmelCase			, __lowerCAmelCase								=					sf.read(__UpperCamelCase	)

           else:
                __lowerCAmelCase			, __lowerCAmelCase								=					sf.read(__UpperCamelCase	)

           __lowerCAmelCase								=					array.T
           if self.mono:
                __lowerCAmelCase								=					librosa.to_mono(__UpperCamelCase	)
           if self.sampling_rate and self.sampling_rate != sampling_rate:
                __lowerCAmelCase								=					librosa.resample(__UpperCamelCase					,						orig_sr=__UpperCamelCase					,						target_sr=self.sampling_rate	)
                __lowerCAmelCase								=					self.sampling_rate

           return {"path": path, "array": array, "sampling_rate": sampling_rate}


      def  __UpperCAmelCase		(	self	)->	Union["FeatureType", Dict[str, "FeatureType"]]:
           from .features import Value

           if self.decode:
                raise ValueError("Cannot flatten a decoded Audio feature."	)
           return {
               "bytes": Value("binary"	),
               "path": Value("string"	),
           }


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	pa.StructArray:
           if pa.types.is_string(storage.type	):
                __lowerCAmelCase								=					pa.array([None] * len(__UpperCamelCase	)					,						type=pa.binary()	)
                __lowerCAmelCase								=					pa.StructArray.from_arrays([bytes_array, storage]					,						["bytes", "path"]					,						mask=storage.is_null()	)
           elif pa.types.is_binary(storage.type	):
                __lowerCAmelCase								=					pa.array([None] * len(__UpperCamelCase	)					,						type=pa.string()	)
                __lowerCAmelCase								=					pa.StructArray.from_arrays([storage, path_array]					,						["bytes", "path"]					,						mask=storage.is_null()	)
           elif pa.types.is_struct(storage.type	) and storage.type.get_all_field_indices("array"	):
                __lowerCAmelCase								=					pa.array([Audio().encode_example(__UpperCamelCase	) if x is not None else None for x in storage.to_pylist()]	)
           elif pa.types.is_struct(storage.type	):
                if storage.type.get_field_index("bytes"	) >= 0:
                     __lowerCAmelCase								=					storage.field("bytes"	)
                else:
                     __lowerCAmelCase								=					pa.array([None] * len(__UpperCamelCase	)					,						type=pa.binary()	)
                if storage.type.get_field_index("path"	) >= 0:
                     __lowerCAmelCase								=					storage.field("path"	)
                else:
                     __lowerCAmelCase								=					pa.array([None] * len(__UpperCamelCase	)					,						type=pa.string()	)
                __lowerCAmelCase								=					pa.StructArray.from_arrays([bytes_array, path_array]					,						["bytes", "path"]					,						mask=storage.is_null()	)
           return array_cast(__UpperCamelCase					,						self.pa_type	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	pa.StructArray:

           @no_op_if_value_is_null
           def path_to_bytes(__UpperCamelCase	):
                with xopen(__UpperCamelCase					,						"rb"	) as f:
                     __lowerCAmelCase								=					f.read()
                return bytes_

           __lowerCAmelCase								=					pa.array(
               [
                   (path_to_bytes(x["path"]	) if x["bytes"] is None else x["bytes"]) if x is not None else None
                   for x in storage.to_pylist()
               ]					,						type=pa.binary()					,						)
           __lowerCAmelCase								=					pa.array(
               [os.path.basename(__UpperCamelCase	) if path is not None else None for path in storage.field("path"	).to_pylist()]					,						type=pa.string()					,						)
           __lowerCAmelCase								=					pa.StructArray.from_arrays([bytes_array, path_array]					,						["bytes", "path"]					,						mask=bytes_array.is_null()	)
           return array_cast(__UpperCamelCase					,						self.pa_type	)














from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase			:					Any         =	logging.get_logger(__name__)

lowerCamelCase			:					Dict         =	{
    '''microsoft/markuplm-base''': '''https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json''',
    '''microsoft/markuplm-large''': '''https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json''',
}



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""markuplm"""


      def __init__(	self					,						__UpperCamelCase=3_0_5_2_2					,						__UpperCamelCase=7_6_8					,						__UpperCamelCase=1_2					,						__UpperCamelCase=1_2					,						__UpperCamelCase=3_0_7_2					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=0.1					,						__UpperCamelCase=0.1					,						__UpperCamelCase=5_1_2					,						__UpperCamelCase=2					,						__UpperCamelCase=0.0_2					,						__UpperCamelCase=1e-12					,						__UpperCamelCase=0					,						__UpperCamelCase=0					,						__UpperCamelCase=2					,						__UpperCamelCase=2_5_6					,						__UpperCamelCase=1_0_2_4					,						__UpperCamelCase=2_1_6					,						__UpperCamelCase=1_0_0_1					,						__UpperCamelCase=3_2					,						__UpperCamelCase=5_0					,						__UpperCamelCase="absolute"					,						__UpperCamelCase=True					,						__UpperCamelCase=None					,						**__UpperCamelCase					,						)->	Dict:
           super().__init__(
               pad_token_id=__UpperCamelCase					,						bos_token_id=__UpperCamelCase					,						eos_token_id=__UpperCamelCase					,						**__UpperCamelCase					,						)
           __lowerCAmelCase								=					vocab_size
           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					hidden_dropout_prob
           __lowerCAmelCase								=					attention_probs_dropout_prob
           __lowerCAmelCase								=					max_position_embeddings
           __lowerCAmelCase								=					type_vocab_size
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					position_embedding_type
           __lowerCAmelCase								=					use_cache
           __lowerCAmelCase								=					classifier_dropout
           # additional properties
           __lowerCAmelCase								=					max_depth
           __lowerCAmelCase								=					max_xpath_tag_unit_embeddings
           __lowerCAmelCase								=					max_xpath_subs_unit_embeddings
           __lowerCAmelCase								=					tag_pad_id
           __lowerCAmelCase								=					subs_pad_id
           __lowerCAmelCase								=					xpath_unit_hidden_size






