





import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
  from .features import FeatureType


_A,       _A,       _A					:				Tuple				=			False, False, False



@dataclass
class 					__snake_case	:


    '''simple docstring'''







    lowerCamelCase__    :		Optional[int]				=	None
    lowerCamelCase__    :		bool				=	True
    lowerCamelCase__    :		bool				=	True
    lowerCamelCase__    :		Optional[str]				=	None
    # Automatically constructed
    lowerCamelCase__    :		ClassVar[str]				=	"dict"
    lowerCamelCase__    :		ClassVar[Any]				=	pa.struct({"""bytes""": pa.binary(), """path""": pa.string()}    )
    lowerCamelCase__    :		str				=	field(default="""Audio"""			,     init=__SCREAMING_SNAKE_CASE			,     repr=__SCREAMING_SNAKE_CASE    )



    def __call__(       self     ):





        '''simple docstring'''

        return self.pa_type



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        try:
            import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
        except ImportError as err:
            raise ImportError('''To support encoding audio data, please install \'soundfile\'.'''     ) from err
        if isinstance(A_   ,				A_     ):
            return {"bytes": None, "path": value}
        elif isinstance(A_   ,				A_     ):
            return {"bytes": value, "path": None}
        elif "array" in value:
            # convert the audio array to wav bytes
            SCREAMING_SNAKE_CASE__											=			BytesIO()
            sf.write(A_   ,				value['''array''']   ,				value['''sampling_rate''']   ,				format='''wav'''     )
            return {"bytes": buffer.getvalue(), "path": None}
        elif value.get('''path'''     ) is not None and os.path.isfile(value['''path''']     ):
            # we set "bytes": None to not duplicate the data if they're already available locally
            if value["path"].endswith('''pcm'''     ):
                # "PCM" only has raw audio bytes
                if value.get('''sampling_rate'''     ) is None:
                    # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
                    raise KeyError('''To use PCM files, please specify a \'sampling_rate\' in Audio object'''     )
                if value.get('''bytes'''     ):
                    # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
                    SCREAMING_SNAKE_CASE__											=			np.frombuffer(value['''bytes''']   ,				dtype=np.intaa     ).astype(np.floataa     ) / 3_27_67
                else:
                    SCREAMING_SNAKE_CASE__											=			np.memmap(value['''path''']   ,				dtype='''h'''   ,				mode='''r'''     ).astype(np.floataa     ) / 3_27_67

                SCREAMING_SNAKE_CASE__											=			BytesIO(bytes()     )
                sf.write(A_   ,				A_   ,				value['''sampling_rate''']   ,				format='''wav'''     )
                return {"bytes": buffer.getvalue(), "path": None}
            else:
                return {"bytes": None, "path": value.get('''path'''     )}
        elif value.get('''bytes'''     ) is not None or value.get('''path'''     ) is not None:
            # store the audio bytes, and path is used to infer the audio format using the file extension
            return {"bytes": value.get('''bytes'''     ), "path": value.get('''path'''     )}
        else:
            raise ValueError(
                f'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''     )



    def  lowercase_					(       self   ,				A_   ,				A_ = None     ):





        '''simple docstring'''

        if not self.decode:
            raise RuntimeError('''Decoding is disabled for this feature. Please use Audio(decode=True) instead.'''     )

        SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			(value['''path'''], BytesIO(value['''bytes''']     )) if value['''bytes'''] is not None else (value['''path'''], None)
        if path is None and file is None:
            raise ValueError(f'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'''     )

        try:
            import librosa
            import soundfile as sf
        except ImportError as err:
            raise ImportError('''To support decoding audio files, please install \'librosa\' and \'soundfile\'.'''     ) from err

        SCREAMING_SNAKE_CASE__											=			xsplitext(A_     )[1][1:].lower() if path is not None else None
        if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
            raise RuntimeError(
                '''Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, '''
                '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '''     )
        elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
            raise RuntimeError(
                '''Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, '''
                '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '''     )

        if file is None:
            SCREAMING_SNAKE_CASE__											=			token_per_repo_id or {}
            SCREAMING_SNAKE_CASE__											=			path.split('''::'''     )[-1]
            try:
                SCREAMING_SNAKE_CASE__											=			string_to_dict(A_   ,				config.HUB_DATASETS_URL     )['''repo_id''']
                SCREAMING_SNAKE_CASE__											=			token_per_repo_id[repo_id]
            except (ValueError, KeyError):
                SCREAMING_SNAKE_CASE__											=			None

            with xopen(A_   ,				'''rb'''   ,				use_auth_token=A_     ) as f:
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			sf.read(A_     )

        else:
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			sf.read(A_     )

        SCREAMING_SNAKE_CASE__											=			array.T
        if self.mono:
            SCREAMING_SNAKE_CASE__											=			librosa.to_mono(A_     )
        if self.sampling_rate and self.sampling_rate != sampling_rate:
            SCREAMING_SNAKE_CASE__											=			librosa.resample(A_   ,				orig_sr=A_   ,				target_sr=self.sampling_rate     )
            SCREAMING_SNAKE_CASE__											=			self.sampling_rate

        return {"path": path, "array": array, "sampling_rate": sampling_rate}



    def  lowercase_					(       self     ):





        '''simple docstring'''

        from .features import Value

        if self.decode:
            raise ValueError('''Cannot flatten a decoded Audio feature.'''     )
        return {
            "bytes": Value('''binary'''     ),
            "path": Value('''string'''     ),
        }



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if pa.types.is_string(storage.type     ):
            SCREAMING_SNAKE_CASE__											=			pa.array([None] * len(A_     )   ,				type=pa.binary()     )
            SCREAMING_SNAKE_CASE__											=			pa.StructArray.from_arrays([bytes_array, storage]   ,				['''bytes''', '''path''']   ,				mask=storage.is_null()     )
        elif pa.types.is_binary(storage.type     ):
            SCREAMING_SNAKE_CASE__											=			pa.array([None] * len(A_     )   ,				type=pa.string()     )
            SCREAMING_SNAKE_CASE__											=			pa.StructArray.from_arrays([storage, path_array]   ,				['''bytes''', '''path''']   ,				mask=storage.is_null()     )
        elif pa.types.is_struct(storage.type     ) and storage.type.get_all_field_indices('''array'''     ):
            SCREAMING_SNAKE_CASE__											=			pa.array([Audio().encode_example(A_     ) if x is not None else None for x in storage.to_pylist()]     )
        elif pa.types.is_struct(storage.type     ):
            if storage.type.get_field_index('''bytes'''     ) >= 0:
                SCREAMING_SNAKE_CASE__											=			storage.field('''bytes'''     )
            else:
                SCREAMING_SNAKE_CASE__											=			pa.array([None] * len(A_     )   ,				type=pa.binary()     )
            if storage.type.get_field_index('''path'''     ) >= 0:
                SCREAMING_SNAKE_CASE__											=			storage.field('''path'''     )
            else:
                SCREAMING_SNAKE_CASE__											=			pa.array([None] * len(A_     )   ,				type=pa.string()     )
            SCREAMING_SNAKE_CASE__											=			pa.StructArray.from_arrays([bytes_array, path_array]   ,				['''bytes''', '''path''']   ,				mask=storage.is_null()     )
        return array_cast(A_   ,				self.pa_type     )




    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        @no_op_if_value_is_null
        def path_to_bytes(A_     ):
            with xopen(A_   ,				'''rb'''     ) as f:
                SCREAMING_SNAKE_CASE__											=			f.read()
            return bytes_

        SCREAMING_SNAKE_CASE__											=			pa.array(
            [
                (path_to_bytes(x['''path''']     ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
                for x in storage.to_pylist()
            ]   ,				type=pa.binary()   ,				)
        SCREAMING_SNAKE_CASE__											=			pa.array(
            [os.path.basename(A_     ) if path is not None else None for path in storage.field('''path'''     ).to_pylist()]   ,				type=pa.string()   ,				)
        SCREAMING_SNAKE_CASE__											=			pa.StructArray.from_arrays([bytes_array, path_array]   ,				['''bytes''', '''path''']   ,				mask=bytes_array.is_null()     )
        return array_cast(A_   ,				self.pa_type     )







"""simple docstring"""





import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


SCREAMING_SNAKE_CASE__   =  logging.getLogger(__name__)

@dataclass
class 						lowercase   :
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42







@dataclass
class 						lowercase   :
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None







class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = 'train'
      _SCREAMING_SNAKE_CASE							      = 'dev'
      _SCREAMING_SNAKE_CASE							      = 'test'







class 						lowercase   :





      @staticmethod
      def        _snake_case     (			lowercase ,		lowercase					)						->	List[InputExample]:
            raise NotImplementedError





      @staticmethod
      def        _snake_case     (			lowercase					)						->	List[str]:
            raise NotImplementedError





      @staticmethod
      def        _snake_case     (			lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase=False ,		lowercase="[CLS]" ,		lowercase=1 ,		lowercase="[SEP]" ,		lowercase=False ,		lowercase=False ,		lowercase=0 ,		lowercase=0 ,		lowercase=-100 ,		lowercase=0 ,		lowercase=True ,		)						->	List[InputFeatures]:
            lowerCAmelCase    		=			{label: i for i, label in enumerate(lowercase					)}

            lowerCAmelCase    		=			[]
            for ex_index, example in enumerate(lowercase					):
                  if ex_index % 10_000 == 0:
                        logger.info("""Writing example %d of %d""" ,		lowercase ,		len(lowercase					)					)

                  lowerCAmelCase    		=			[]
                  lowerCAmelCase    		=			[]
                  for word, label in zip(example.words ,		example.labels					):
                        lowerCAmelCase    		=			tokenizer.tokenize(lowercase					)

                        # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
                        if len(lowercase					) > 0:
                              tokens.extend(lowercase					)
                              # Use the real label id for the first token of the word, and padding ids for the remaining tokens
                              label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(lowercase					) - 1)					)

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
                  lowerCAmelCase    		=			tokenizer.num_special_tokens_to_add()
                  if len(lowercase					) > max_seq_length - special_tokens_count:
                        lowerCAmelCase    		=			tokens[: (max_seq_length - special_tokens_count)]
                        lowerCAmelCase    		=			label_ids[: (max_seq_length - special_tokens_count)]

                  # The convention in BERT is:
                  # (a) For sequence pairs:
                  #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
                  #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
                  # (b) For single sequences:
                  #  tokens:   [CLS] the dog is hairy . [SEP]
                  #  type_ids:   0   0   0   0  0     0   0
                  #
                  # Where "type_ids" are used to indicate whether this is the first
                  # sequence or the second sequence. The embedding vectors for `type=0` and
                  # `type=1` were learned during pre-training and are added to the wordpiece
                  # embedding vector (and position vector). This is not *strictly* necessary
                  # since the [SEP] token unambiguously separates the sequences, but it makes
                  # it easier for the model to learn the concept of sequences.
                  #
                  # For classification tasks, the first vector (corresponding to [CLS]) is
                  # used as the "sentence vector". Note that this only makes sense because
                  # the entire model is fine-tuned.
                  tokens += [sep_token]
                  label_ids += [pad_token_label_id]
                  if sep_token_extra:
                        # roberta uses an extra separator b/w pairs of sentences
                        tokens += [sep_token]
                        label_ids += [pad_token_label_id]
                  lowerCAmelCase    		=			[sequence_a_segment_id] * len(lowercase					)

                  if cls_token_at_end:
                        tokens += [cls_token]
                        label_ids += [pad_token_label_id]
                        segment_ids += [cls_token_segment_id]
                  else:
                        lowerCAmelCase    		=			[cls_token] + tokens
                        lowerCAmelCase    		=			[pad_token_label_id] + label_ids
                        lowerCAmelCase    		=			[cls_token_segment_id] + segment_ids

                  lowerCAmelCase    		=			tokenizer.convert_tokens_to_ids(lowercase					)

                  # The mask has 1 for real tokens and 0 for padding tokens. Only real
                  # tokens are attended to.
                  lowerCAmelCase    		=			[1 if mask_padding_with_zero else 0] * len(lowercase					)

                  # Zero-pad up to the sequence length.
                  lowerCAmelCase    		=			max_seq_length - len(lowercase					)
                  if pad_on_left:
                        lowerCAmelCase    		=			([pad_token] * padding_length) + input_ids
                        lowerCAmelCase    		=			([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
                        lowerCAmelCase    		=			([pad_token_segment_id] * padding_length) + segment_ids
                        lowerCAmelCase    		=			([pad_token_label_id] * padding_length) + label_ids
                  else:
                        input_ids += [pad_token] * padding_length
                        input_mask += [0 if mask_padding_with_zero else 1] * padding_length
                        segment_ids += [pad_token_segment_id] * padding_length
                        label_ids += [pad_token_label_id] * padding_length

                  assert len(lowercase					) == max_seq_length
                  assert len(lowercase					) == max_seq_length
                  assert len(lowercase					) == max_seq_length
                  assert len(lowercase					) == max_seq_length

                  if ex_index < 5:
                        logger.info("""*** Example ***"""					)
                        logger.info("""guid: %s""" ,		example.guid					)
                        logger.info("""tokens: %s""" ,		""" """.join([str(lowercase					) for x in tokens]					)					)
                        logger.info("""input_ids: %s""" ,		""" """.join([str(lowercase					) for x in input_ids]					)					)
                        logger.info("""input_mask: %s""" ,		""" """.join([str(lowercase					) for x in input_mask]					)					)
                        logger.info("""segment_ids: %s""" ,		""" """.join([str(lowercase					) for x in segment_ids]					)					)
                        logger.info("""label_ids: %s""" ,		""" """.join([str(lowercase					) for x in label_ids]					)					)

                  if "token_type_ids" not in tokenizer.model_input_names:
                        lowerCAmelCase    		=			None

                  features.append(
                      InputFeatures(
                          input_ids=lowercase ,		attention_mask=lowercase ,		token_type_ids=lowercase ,		label_ids=lowercase					)					)
            return features


if is_torch_available():
       import torch
       from torch import nn
       from torch.utils.data import Dataset

       class 						lowercase   (   _UpperCAmelCase   ):
             _SCREAMING_SNAKE_CASE							      = 42
             _SCREAMING_SNAKE_CASE							      = nn.CrossEntropyLoss().ignore_index





             def __init__(			self ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase = None ,		lowercase=False ,		lowercase = Split.train ,		)						->	List[str]:
                   # Load data features from cache or dataset file
                   lowerCAmelCase    		=			os.path.join(
                       lowercase ,		"""cached_{}_{}_{}""".format(mode.value ,		tokenizer.__class__.__name__ ,		str(lowercase					)					) ,		)

                   # Make sure only the first process in distributed training processes the dataset,
                   # and the others will use the cache.
                   lowerCAmelCase    		=			cached_features_file + """.lock"""
                   with FileLock(lowercase					):
                         if os.path.exists(lowercase					) and not overwrite_cache:
                               logger.info(f'Loading features from cached file {cached_features_file}'					)
                               lowerCAmelCase    		=			torch.load(lowercase					)
                         else:
                               logger.info(f'Creating features from dataset file at {data_dir}'					)
                               lowerCAmelCase    		=			token_classification_task.read_examples_from_file(lowercase ,		lowercase					)
                               # TODO clean up all this to leverage built-in features of tokenizers
                               lowerCAmelCase    		=			token_classification_task.convert_examples_to_features(
                                   lowercase ,		lowercase ,		lowercase ,		lowercase ,		cls_token_at_end=bool(model_type in ["""xlnet"""]					) ,		cls_token=tokenizer.cls_token ,		cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0 ,		sep_token=tokenizer.sep_token ,		sep_token_extra=lowercase ,		pad_on_left=bool(tokenizer.padding_side == """left"""					) ,		pad_token=tokenizer.pad_token_id ,		pad_token_segment_id=tokenizer.pad_token_type_id ,		pad_token_label_id=self.pad_token_label_id ,		)
                               logger.info(f'Saving features into cached file {cached_features_file}'					)
                               torch.save(self.features ,		lowercase					)





             def __len__(			self					)						->	int:
                   return len(self.features					)





             def __getitem__(			self ,		lowercase					)						->	InputFeatures:
                   return self.features[i]


if is_tf_available():
       import tensorflow as tf

       class 						lowercase   :
             _SCREAMING_SNAKE_CASE							      = 42
             _SCREAMING_SNAKE_CASE							      = -100





             def __init__(			self ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase = None ,		lowercase=False ,		lowercase = Split.train ,		)						->	Any:
                   lowerCAmelCase    		=			token_classification_task.read_examples_from_file(lowercase ,		lowercase					)
                   # TODO clean up all this to leverage built-in features of tokenizers
                   lowerCAmelCase    		=			token_classification_task.convert_examples_to_features(
                       lowercase ,		lowercase ,		lowercase ,		lowercase ,		cls_token_at_end=bool(model_type in ["""xlnet"""]					) ,		cls_token=tokenizer.cls_token ,		cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0 ,		sep_token=tokenizer.sep_token ,		sep_token_extra=lowercase ,		pad_on_left=bool(tokenizer.padding_side == """left"""					) ,		pad_token=tokenizer.pad_token_id ,		pad_token_segment_id=tokenizer.pad_token_type_id ,		pad_token_label_id=self.pad_token_label_id ,		)

                   def gen():
                         for ex in self.features:
                               if ex.token_type_ids is None:
                                     yield (
                                         {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                                         ex.label_ids,
                                     )
                               else:
                                     yield (
                                         {
                                             "input_ids": ex.input_ids,
                                             "attention_mask": ex.attention_mask,
                                             "token_type_ids": ex.token_type_ids,
                                         },
                                         ex.label_ids,
                                     )

                   if "token_type_ids" not in tokenizer.model_input_names:
                         lowerCAmelCase    		=			tf.data.Dataset.from_generator(
                             lowercase ,		({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa}, tf.intaa) ,		(
                                 {"""input_ids""": tf.TensorShape([None]					), """attention_mask""": tf.TensorShape([None]					)},
                                 tf.TensorShape([None]					),
                             ) ,		)
                   else:
                         lowerCAmelCase    		=			tf.data.Dataset.from_generator(
                             lowercase ,		({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa, """token_type_ids""": tf.intaa}, tf.intaa) ,		(
                                 {
                                     """input_ids""": tf.TensorShape([None]					),
                                     """attention_mask""": tf.TensorShape([None]					),
                                     """token_type_ids""": tf.TensorShape([None]					),
                                 },
                                 tf.TensorShape([None]					),
                             ) ,		)





             def        _snake_case     (			self					)						->	List[str]:
                   lowerCAmelCase    		=			self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features					)					)					)

                   return self.dataset





             def __len__(			self					)						->	Optional[int]:
                   return len(self.features					)





             def __getitem__(			self ,		lowercase					)						->	InputFeatures:
                   return self.features[i]






