


'''simple docstring'''



import os
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Dict, List, Optional, Union

import torch
from filelock import FileLock
from torch.utils.data import Dataset

from ...models.auto.modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
from ..processors.squad import SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features


__A		=logging.get_logger(__name__)

__A		=list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
__A		=tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)


@dataclass
class 						_snake_case   :
					lowerCAmelCase :str						=   field(
					    default=a__    ,				metadata={'''help''': '''Model type selected in the list: ''' + ''', '''.join(a__		)}		)
					lowerCAmelCase :str						=   field(
					    default=a__    ,				metadata={'''help''': '''The input data dir. Should contain the .json files for the SQuAD task.'''}		)
					lowerCAmelCase :int						=   field(
					    default=128    ,				metadata={
					        '''help''': (
					            '''The maximum total input sequence length after tokenization. Sequences longer '''
					            '''than this will be truncated, sequences shorter will be padded.'''
					        )
					    }    ,				)
					lowerCAmelCase :int						=   field(
					    default=128    ,				metadata={'''help''': '''When splitting up a long document into chunks, how much stride to take between chunks.'''}    ,				)
					lowerCAmelCase :int						=   field(
					    default=64    ,				metadata={
					        '''help''': (
					            '''The maximum number of tokens for the question. Questions longer than this will '''
					            '''be truncated to this length.'''
					        )
					    }    ,				)
					lowerCAmelCase :int						=   field(
					    default=30    ,				metadata={
					        '''help''': (
					            '''The maximum length of an answer that can be generated. This is needed because the start '''
					            '''and end predictions are not conditioned on one another.'''
					        )
					    }    ,				)
					lowerCAmelCase :bool						=   field(
					    default=a__    ,				metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}		)
					lowerCAmelCase :bool						=   field(
					    default=a__    ,				metadata={'''help''': '''If true, the SQuAD examples contain some that do not have an answer.'''}		)
					lowerCAmelCase :float						=   field(
					    default=0.0    ,				metadata={'''help''': '''If null_score - best_non_null is greater than the threshold predict null.'''}		)
					lowerCAmelCase :int						=   field(
					    default=20    ,				metadata={'''help''': '''If null_score - best_non_null is greater than the threshold predict null.'''}		)
					lowerCAmelCase :int						=   field(
					    default=0    ,				metadata={
					        '''help''': (
					            '''language id of input for language-specific xlm models (see'''
					            ''' tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)'''
					        )
					    }    ,				)
					lowerCAmelCase :int						=   field(default=1    ,				metadata={'''help''': '''multiple threads for converting example to features'''}		)


class 						_snake_case   (       a__		):
					lowerCAmelCase :Tuple						=   'train'
					lowerCAmelCase :List[Any]						=   'dev'


class 						_snake_case   (       a__		):
					lowerCAmelCase :SquadDataTrainingArguments
					lowerCAmelCase :List[SquadFeatures]
					lowerCAmelCase :Split
					lowerCAmelCase :bool




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase = None							,							_lowerCamelCase = Split.train							,							_lowerCamelCase = False							,							_lowerCamelCase = None							,							_lowerCamelCase = "pt"							,							):
								UpperCAmelCase__					:   Any       				=       args
								UpperCAmelCase__					:   Tuple       				=       is_language_sensitive
								UpperCAmelCase__					:   List[str]       				=       SquadVaProcessor() if args.version_2_with_negative else SquadVaProcessor()
								if isinstance(_lowerCamelCase							,							_lowerCamelCase):
											try:
														UpperCAmelCase__					:   Tuple       				=       Split[mode]
											except KeyError:
														raise KeyError("""mode is not a valid split name""")
								UpperCAmelCase__					:   List[str]       				=       mode
								# Load data features from cache or dataset file
								UpperCAmelCase__					:   int       				=       '''v2''' if args.version_2_with_negative else '''v1'''
								UpperCAmelCase__					:   str       				=       os.path.join(
								    cache_dir if cache_dir is not None else args.data_dir							,							f'''cached_{mode.value}_{tokenizer.__class__.__name__}_{args.max_seq_length}_{version_tag}'''							,							)

								# Make sure only the first process in distributed training processes the dataset,
								# and the others will use the cache.
								UpperCAmelCase__					:   Tuple       				=       cached_features_file + '''.lock'''
								with FileLock(_lowerCamelCase):
											if os.path.exists(_lowerCamelCase) and not args.overwrite_cache:
														UpperCAmelCase__					:   List[Any]       				=       time.time()
														UpperCAmelCase__					:   Tuple       				=       torch.load(_lowerCamelCase)

														# Legacy cache files have only features, while new cache files
														# will have dataset and examples also.
														UpperCAmelCase__					:   Union[str, Any]       				=       self.old_features['''features''']
														UpperCAmelCase__					:   Optional[int]       				=       self.old_features.get("""dataset"""							,							_lowerCamelCase)
														UpperCAmelCase__					:   List[str]       				=       self.old_features.get("""examples"""							,							_lowerCamelCase)
														logger.info(
														    f'''Loading features from cached file {cached_features_file} [took %.3f s]'''							,							time.time() - start)

														if self.dataset is None or self.examples is None:
																	logger.warning(
																	    f'''Deleting cached file {cached_features_file} will allow dataset and examples to be cached in'''
																	    """ future run""")
											else:
														if mode == Split.dev:
																	UpperCAmelCase__					:   int       				=       self.processor.get_dev_examples(args.data_dir)
														else:
																	UpperCAmelCase__					:   int       				=       self.processor.get_train_examples(args.data_dir)

														UpperCAmelCase__					:   Any       				=       squad_convert_examples_to_features(
														    examples=self.examples							,							tokenizer=_lowerCamelCase							,							max_seq_length=args.max_seq_length							,							doc_stride=args.doc_stride							,							max_query_length=args.max_query_length							,							is_training=mode == Split.train							,							threads=args.threads							,							return_dataset=_lowerCamelCase							,							)

														UpperCAmelCase__					:   str       				=       time.time()
														torch.save(
														    {"""features""": self.features, """dataset""": self.dataset, """examples""": self.examples}							,							_lowerCamelCase							,							)
														# ^ This seems to take a lot of time so I want to investigate why and how we can improve.
														logger.info(
														    f'''Saving features into cached file {cached_features_file} [took {time.time() - start:.3f} s]''')




					def __len__( self):
								return len(self.features)




					def __getitem__( self							,							_lowerCamelCase):
								# Convert to Tensors and build dataset
								UpperCAmelCase__					:   List[Any]       				=       self.features[i]

								UpperCAmelCase__					:   List[str]       				=       torch.tensor(feature.input_ids							,							dtype=torch.long)
								UpperCAmelCase__					:   str       				=       torch.tensor(feature.attention_mask							,							dtype=torch.long)
								UpperCAmelCase__					:   List[str]       				=       torch.tensor(feature.token_type_ids							,							dtype=torch.long)
								UpperCAmelCase__					:   List[Any]       				=       torch.tensor(feature.cls_index							,							dtype=torch.long)
								UpperCAmelCase__					:   List[str]       				=       torch.tensor(feature.p_mask							,							dtype=torch.float)
								UpperCAmelCase__					:   Dict       				=       torch.tensor(feature.is_impossible							,							dtype=torch.float)

								UpperCAmelCase__					:   Any       				=       {
								    '''input_ids''': input_ids,
								    '''attention_mask''': attention_mask,
								    '''token_type_ids''': token_type_ids,
								}

								if self.args.model_type in ["xlm", "roberta", "distilbert", "camembert"]:
											del inputs["token_type_ids"]

								if self.args.model_type in ["xlnet", "xlm"]:
											inputs.update({"""cls_index""": cls_index, """p_mask""": p_mask})
											if self.args.version_2_with_negative:
														inputs.update({"""is_impossible""": is_impossible})
											if self.is_language_sensitive:
														inputs.update({"""langs""": (torch.ones(input_ids.shape							,							dtype=torch.intaa) * self.args.lang_id)})

								if self.mode == Split.train:
											UpperCAmelCase__					:   List[str]       				=       torch.tensor(feature.start_position							,							dtype=torch.long)
											UpperCAmelCase__					:   str       				=       torch.tensor(feature.end_position							,							dtype=torch.long)
											inputs.update({"""start_positions""": start_positions, """end_positions""": end_positions})

								return inputs



'''simple docstring'''



import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
		from .features import FeatureType


__A		,	__A		,	__A		=False, False, False


@dataclass
class 						_snake_case   :
					lowerCAmelCase :Optional[int]						=   None
					lowerCAmelCase :bool						=   True
					lowerCAmelCase :bool						=   True
					lowerCAmelCase :Optional[str]						=   None
					# Automatically constructed
					lowerCAmelCase :ClassVar[str]						=   "dict"
					lowerCAmelCase :ClassVar[Any]						=   pa.struct({'''bytes''': pa.binary(), '''path''': pa.string()}		)
					lowerCAmelCase :str						=   field(default='''Audio'''    ,				init=a__    ,				repr=a__		)




					def __call__( self):
								return self.pa_type




					def 			snake_case__  ( self							,							_lowerCamelCase):
								try:
											import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
								except ImportError as err:
											raise ImportError("""To support encoding audio data, please install 'soundfile'.""") from err
								if isinstance(_lowerCamelCase							,							_lowerCamelCase):
											return {"bytes": None, "path": value}
								elif isinstance(_lowerCamelCase							,							_lowerCamelCase):
											return {"bytes": value, "path": None}
								elif "array" in value:
											# convert the audio array to wav bytes
											UpperCAmelCase__					:   Optional[int]       				=       BytesIO()
											sf.write(_lowerCamelCase							,							value["""array"""]							,							value["""sampling_rate"""]							,							format="""wav""")
											return {"bytes": buffer.getvalue(), "path": None}
								elif value.get("""path""") is not None and os.path.isfile(value["""path"""]):
											# we set "bytes": None to not duplicate the data if they're already available locally
											if value["path"].endswith("""pcm"""):
														# "PCM" only has raw audio bytes
														if value.get("""sampling_rate""") is None:
																	# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
																	raise KeyError("""To use PCM files, please specify a 'sampling_rate' in Audio object""")
														if value.get("""bytes"""):
																	# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
																	UpperCAmelCase__					:   Tuple       				=       np.frombuffer(value["""bytes"""]							,							dtype=np.intaa).astype(np.floataa) / 3_2767
														else:
																	UpperCAmelCase__					:   List[str]       				=       np.memmap(value["""path"""]							,							dtype="""h"""							,							mode="""r""").astype(np.floataa) / 3_2767

														UpperCAmelCase__					:   List[str]       				=       BytesIO(bytes())
														sf.write(_lowerCamelCase							,							_lowerCamelCase							,							value["""sampling_rate"""]							,							format="""wav""")
														return {"bytes": buffer.getvalue(), "path": None}
											else:
														return {"bytes": None, "path": value.get("""path""")}
								elif value.get("""bytes""") is not None or value.get("""path""") is not None:
											# store the audio bytes, and path is used to infer the audio format using the file extension
											return {"bytes": value.get("""bytes"""), "path": value.get("""path""")}
								else:
											raise ValueError(
											    f'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.''')




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if not self.decode:
											raise RuntimeError("""Decoding is disabled for this feature. Please use Audio(decode=True) instead.""")

								UpperCAmelCase__				,    UpperCAmelCase__					:   Tuple       				=       (value["""path"""], BytesIO(value["""bytes"""])) if value["""bytes"""] is not None else (value["""path"""], None)
								if path is None and file is None:
											raise ValueError(f'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.''')

								try:
											import librosa
											import soundfile as sf
								except ImportError as err:
											raise ImportError("""To support decoding audio files, please install 'librosa' and 'soundfile'.""") from err

								UpperCAmelCase__					:   Dict       				=       xsplitext(_lowerCamelCase)[1][1:].lower() if path is not None else None
								if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
											raise RuntimeError(
											    """Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, """
											    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """)
								elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
											raise RuntimeError(
											    """Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, """
											    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """)

								if file is None:
											UpperCAmelCase__					:   int       				=       token_per_repo_id or {}
											UpperCAmelCase__					:   Optional[int]       				=       path.split("""::""")[-1]
											try:
														UpperCAmelCase__					:   Dict       				=       string_to_dict(_lowerCamelCase							,							config.HUB_DATASETS_URL)["""repo_id"""]
														UpperCAmelCase__					:   Dict       				=       token_per_repo_id[repo_id]
											except (ValueError, KeyError):
														UpperCAmelCase__					:   List[Any]       				=       None

											with xopen(_lowerCamelCase							,							"""rb"""							,							use_auth_token=_lowerCamelCase) as f:
														UpperCAmelCase__				,    UpperCAmelCase__					:   List[str]       				=       sf.read(_lowerCamelCase)

								else:
											UpperCAmelCase__				,    UpperCAmelCase__					:   List[Any]       				=       sf.read(_lowerCamelCase)

								UpperCAmelCase__					:   str       				=       array.T
								if self.mono:
											UpperCAmelCase__					:   List[Any]       				=       librosa.to_mono(_lowerCamelCase)
								if self.sampling_rate and self.sampling_rate != sampling_rate:
											UpperCAmelCase__					:   int       				=       librosa.resample(_lowerCamelCase							,							orig_sr=_lowerCamelCase							,							target_sr=self.sampling_rate)
											UpperCAmelCase__					:   Tuple       				=       self.sampling_rate

								return {"path": path, "array": array, "sampling_rate": sampling_rate}




					def 			snake_case__  ( self):
								from .features import Value

								if self.decode:
											raise ValueError("""Cannot flatten a decoded Audio feature.""")
								return {
								    "bytes": Value("""binary"""),
								    "path": Value("""string"""),
								}




					def 			snake_case__  ( self							,							_lowerCamelCase):
								if pa.types.is_string(storage.type):
											UpperCAmelCase__					:   Dict       				=       pa.array([None] * len(_lowerCamelCase)							,							type=pa.binary())
											UpperCAmelCase__					:   Tuple       				=       pa.StructArray.from_arrays([bytes_array, storage]							,							["""bytes""", """path"""]							,							mask=storage.is_null())
								elif pa.types.is_binary(storage.type):
											UpperCAmelCase__					:   Optional[int]       				=       pa.array([None] * len(_lowerCamelCase)							,							type=pa.string())
											UpperCAmelCase__					:   str       				=       pa.StructArray.from_arrays([storage, path_array]							,							["""bytes""", """path"""]							,							mask=storage.is_null())
								elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("""array"""):
											UpperCAmelCase__					:   Optional[Any]       				=       pa.array([Audio().encode_example(_lowerCamelCase) if x is not None else None for x in storage.to_pylist()])
								elif pa.types.is_struct(storage.type):
											if storage.type.get_field_index("""bytes""") >= 0:
														UpperCAmelCase__					:   int       				=       storage.field("""bytes""")
											else:
														UpperCAmelCase__					:   List[str]       				=       pa.array([None] * len(_lowerCamelCase)							,							type=pa.binary())
											if storage.type.get_field_index("""path""") >= 0:
														UpperCAmelCase__					:   List[Any]       				=       storage.field("""path""")
											else:
														UpperCAmelCase__					:   Optional[int]       				=       pa.array([None] * len(_lowerCamelCase)							,							type=pa.string())
											UpperCAmelCase__					:   Optional[int]       				=       pa.StructArray.from_arrays([bytes_array, path_array]							,							["""bytes""", """path"""]							,							mask=storage.is_null())
								return array_cast(_lowerCamelCase							,							self.pa_type)




					def 			snake_case__  ( self							,							_lowerCamelCase):

								@no_op_if_value_is_null
								def path_to_bytes(_lowerCamelCase):
											with xopen(_lowerCamelCase							,							"""rb""") as f:
														UpperCAmelCase__					:   int       				=       f.read()
											return bytes_

								UpperCAmelCase__					:   Optional[Any]       				=       pa.array(
								    [
								        (path_to_bytes(x["""path"""]) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
								        for x in storage.to_pylist()
								    ]							,							type=pa.binary()							,							)
								UpperCAmelCase__					:   Optional[Any]       				=       pa.array(
								    [os.path.basename(_lowerCamelCase) if path is not None else None for path in storage.field("""path""").to_pylist()]							,							type=pa.string()							,							)
								UpperCAmelCase__					:   List[str]       				=       pa.StructArray.from_arrays([bytes_array, path_array]							,							["""bytes""", """path"""]							,							mask=bytes_array.is_null())
								return array_cast(_lowerCamelCase							,							self.pa_type)