





from typing import List, Optional

import numpy as np

from ...processing_utils import ProcessorMixin
from ...utils import to_numpy

class SCREAMING_SNAKE_CASE_ (					_snake_case   ):





					"""simple docstring"""


					A__				 =    """EncodecFeatureExtractor"""
					A__				 =    ("""T5Tokenizer""", """T5TokenizerFast""")





					def __init__( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										super().__init__(snake_case_	,  snake_case_	)
										lowerCamelCase__       =       self.feature_extractor
										lowerCamelCase__       =       False





					def 					__magic_name__ ( self	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=True	):
										return self.tokenizer.get_decoder_prompt_ids(task=snake_case_	,  language=snake_case_	,  no_timestamps=snake_case_	)





					def __call__( self	,  *_lowerCAmelCase	,  **_lowerCAmelCase	):
										# For backward compatibility
										if self._in_target_context_manager:
															return self.current_processor(*snake_case_	,  **snake_case_	)

										lowerCamelCase__       =       kwargs.pop("audio"	,  snake_case_	)
										lowerCamelCase__       =       kwargs.pop("sampling_rate"	,  snake_case_	)
										lowerCamelCase__       =       kwargs.pop("text"	,  snake_case_	)
										if len(snake_case_	) > 0:
															lowerCamelCase__       =       args[0]
															lowerCamelCase__       =       args[1:]

										if audio is None and text is None:
															raise ValueError("You need to specify either an `audio` or `text` input to process."	)

										if text is not None:
															lowerCamelCase__       =       self.tokenizer(snake_case_	,  **snake_case_	)

										if audio is not None:
															lowerCamelCase__       =       self.feature_extractor(snake_case_	,  *snake_case_	,  sampling_rate=snake_case_	,  **snake_case_	)

										if audio is None:
															return inputs

										elif text is None:
															return audio_inputs

										else:
															lowerCamelCase__       =       audio_inputs["input_values"]
															if "padding_mask" in audio_inputs:
																				lowerCamelCase__       =       audio_inputs["padding_mask"]
															return inputs





					def 					__magic_name__ ( self	,  *_lowerCAmelCase	,  **_lowerCAmelCase	):
										lowerCamelCase__       =       kwargs.pop("audio"	,  snake_case_	)
										lowerCamelCase__       =       kwargs.pop("padding_mask"	,  snake_case_	)

										if len(snake_case_	) > 0:
															lowerCamelCase__       =       args[0]
															lowerCamelCase__       =       args[1:]

										if audio_values is not None:
															return self._decode_audio(snake_case_	,  padding_mask=snake_case_	)
										else:
															return self.tokenizer.batch_decode(*snake_case_	,  **snake_case_	)





					def 					__magic_name__ ( self	,  *_lowerCAmelCase	,  **_lowerCAmelCase	):
										return self.tokenizer.decode(*snake_case_	,  **snake_case_	)






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase = None	):
										lowerCamelCase__       =       to_numpy(snake_case_	)
										lowerCamelCase__       =       audio_values.shape

										if padding_mask is None:
															return list(snake_case_	)

										lowerCamelCase__       =       to_numpy(snake_case_	)

										# match the sequence length of the padding mask to the generated audio arrays by padding with the **non-padding**
										# token (so that the generated audio values are **not** treated as padded tokens)
										lowerCamelCase__       =       seq_len - padding_mask.shape[-1]
										lowerCamelCase__       =       1 - self.feature_extractor.padding_value
										lowerCamelCase__       =       np.pad(snake_case_	,  ((0, 0), (0, difference))	,  "constant"	,  constant_values=snake_case_	)

										lowerCamelCase__       =       audio_values.tolist()
										for i in range(snake_case_	):
															lowerCamelCase__       =       np.asarray(audio_values[i]	)[
															    padding_mask[i][None, :] != self.feature_extractor.padding_value
															]
															lowerCamelCase__       =       sliced_audio.reshape(snake_case_	,  -1	)

										return audio_values








import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class SCREAMING_SNAKE_CASE_ (					datasets.BeamBasedBuilder   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	):
										return datasets.DatasetInfo(
										    features=datasets.Features({"content": datasets.Value("string"	)}	)	,  supervised_keys=_lowerCAmelCase	,  )





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,  gen_kwargs={"examples": get_test_dummy_examples()}	)]






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										import apache_beam as beam

										return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase	)

class SCREAMING_SNAKE_CASE_ (					datasets.BeamBasedBuilder   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	):
										return datasets.DatasetInfo(
										    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"	)}	)}	)	,  supervised_keys=_lowerCAmelCase	,  )





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										return [
										    datasets.SplitGenerator(name=datasets.Split.TRAIN	,  gen_kwargs={"examples": get_test_nested_examples()}	)
										]






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										import apache_beam as beam

										return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase	)





def    __UpperCamelCase      (    ) ->Optional[Any]:
					return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"])]





def    __UpperCamelCase      (    ) ->Optional[Any]:
					return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"])]







class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					@require_beam
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       len(get_test_dummy_examples()	)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       DummyBeamDataset(cache_dir=_lowerCAmelCase	,  beam_runner="DirectRunner"	)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train.arrow"	)	)	)
															self.assertDictEqual(builder.info.features	,  datasets.Features({"content": datasets.Value("string"	)}	)	)
															lowerCamelCase__       =       builder.as_dataset()
															self.assertEqual(dset["train"].num_rows	,  _lowerCAmelCase	)
															self.assertEqual(dset["train"].info.splits["train"].num_examples	,  _lowerCAmelCase	)
															self.assertDictEqual(dset["train"][0]	,  get_test_dummy_examples()[0][1]	)
															self.assertDictEqual(
															    dset["train"][expected_num_examples - 1]	,  get_test_dummy_examples()[expected_num_examples - 1][1]	)
															self.assertTrue(
															    os.path.exists(os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  "dataset_info.json"	)	)	)
															del dset





					@require_beam
					def 					__magic_name__ ( self	):
										import apache_beam as beam

										lowerCamelCase__       =       beam.io.parquetio.WriteToParquet

										lowerCamelCase__       =       len(get_test_dummy_examples()	)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       DummyBeamDataset(cache_dir=_lowerCAmelCase	,  beam_runner="DirectRunner"	)
															with patch("apache_beam.io.parquetio.WriteToParquet"	) as write_parquet_mock:
																				lowerCamelCase__       =       partial(_lowerCAmelCase	,  num_shards=2	)
																				builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            _lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train-00000-of-00002.arrow"	)	)	)
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            _lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train-00000-of-00002.arrow"	)	)	)
															self.assertDictEqual(builder.info.features	,  datasets.Features({"content": datasets.Value("string"	)}	)	)
															lowerCamelCase__       =       builder.as_dataset()
															self.assertEqual(dset["train"].num_rows	,  _lowerCAmelCase	)
															self.assertEqual(dset["train"].info.splits["train"].num_examples	,  _lowerCAmelCase	)
															# Order is not preserved when sharding, so we just check that all the elements are there
															self.assertListEqual(sorted(dset["train"]["content"]	)	,  sorted(["foo", "bar", "foobar"]	)	)
															self.assertTrue(
															    os.path.exists(os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  "dataset_info.json"	)	)	)
															del dset





					@require_beam
					def 					__magic_name__ ( self	):
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       DummyBeamDataset(cache_dir=_lowerCAmelCase	)
															self.assertRaises(datasets.builder.MissingBeamOptions	,  builder.download_and_prepare	)






					@require_beam
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       len(get_test_nested_examples()	)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       NestedBeamDataset(cache_dir=_lowerCAmelCase	,  beam_runner="DirectRunner"	)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train.arrow"	)	)	)
															self.assertDictEqual(
															    builder.info.features	,  datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"	)}	)}	)	)
															lowerCamelCase__       =       builder.as_dataset()
															self.assertEqual(dset["train"].num_rows	,  _lowerCAmelCase	)
															self.assertEqual(dset["train"].info.splits["train"].num_examples	,  _lowerCAmelCase	)
															self.assertDictEqual(dset["train"][0]	,  get_test_nested_examples()[0][1]	)
															self.assertDictEqual(
															    dset["train"][expected_num_examples - 1]	,  get_test_nested_examples()[expected_num_examples - 1][1]	)
															self.assertTrue(
															    os.path.exists(os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  "dataset_info.json"	)	)	)
															del dset

