






import random
import shutil
import tempfile
import unittest

import numpy as np

from transformers import TaTokenizer, TaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torch
from transformers.utils.import_utils import is_speech_available, is_torch_available


if is_torch_available():
   pass

if is_speech_available():
   from transformers import EncodecFeatureExtractor, MusicgenProcessor


snake_case			 =						random.Random()


def      lowerCamelCase__	(   lowercase  ,   lowercase=1.0  ,   lowercase=None  ,   lowercase=None ):

  """simple docstring"""






  if rng is None:
    SCREAMING_SNAKE_CASE				:					Tuple		  =			global_rng

  SCREAMING_SNAKE_CASE				:					Any		  =			[]
  for batch_idx in range(shape[0] ):
    values.append([] )
    for _ in range(shape[1] ):
      values[-1].append(rng.random() * scale )

  return values


@require_torch
@require_sentencepiece
class        SCREAMING_SNAKE_CASE	(	unittest.TestCase    ):
     '''simple docstring'''


     def  _A				( self				:		List[str]					):
       SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			"facebook/musicgen-small"
       SCREAMING_SNAKE_CASE				:					Dict		  =			tempfile.mkdtemp()


     def  _A				( self				:		str    ,       **UpperCAmelCase_				:		List[str]					):
       return TaTokenizer.from_pretrained(self.checkpoint    ,       **UpperCAmelCase_					)


     def  _A				( self				:		Dict    ,       **UpperCAmelCase_				:		int					):
       return EncodecFeatureExtractor.from_pretrained(self.checkpoint    ,       **UpperCAmelCase_					)


     def  _A				( self				:		str					):
       shutil.rmtree(self.tmpdirname					)


     def  _A				( self				:		Optional[Any]					):
       SCREAMING_SNAKE_CASE				:					str		  =			self.get_tokenizer()
       SCREAMING_SNAKE_CASE				:					Any		  =			self.get_feature_extractor()

       SCREAMING_SNAKE_CASE				:					Tuple		  =			MusicgenProcessor(tokenizer=UpperCAmelCase_    ,       feature_extractor=UpperCAmelCase_					)

       processor.save_pretrained(self.tmpdirname					)
       SCREAMING_SNAKE_CASE				:					int		  =			MusicgenProcessor.from_pretrained(self.tmpdirname					)

       self.assertEqual(processor.tokenizer.get_vocab()    ,       tokenizer.get_vocab()					)
       self.assertIsInstance(processor.tokenizer    ,       UpperCAmelCase_					)

       self.assertEqual(processor.feature_extractor.to_json_string()    ,       feature_extractor.to_json_string()					)
       self.assertIsInstance(processor.feature_extractor    ,       UpperCAmelCase_					)


     def  _A				( self				:		Any					):
       SCREAMING_SNAKE_CASE				:					List[str]		  =			MusicgenProcessor(tokenizer=self.get_tokenizer()    ,       feature_extractor=self.get_feature_extractor()					)
       processor.save_pretrained(self.tmpdirname					)

       SCREAMING_SNAKE_CASE				:					Tuple		  =			self.get_tokenizer(bos_token="(BOS)"    ,       eos_token="(EOS)"					)
       SCREAMING_SNAKE_CASE				:					str		  =			self.get_feature_extractor(do_normalize=UpperCAmelCase_    ,       padding_value=1.0					)

       SCREAMING_SNAKE_CASE				:					Any		  =			MusicgenProcessor.from_pretrained(
           self.tmpdirname    ,       bos_token="(BOS)"    ,       eos_token="(EOS)"    ,       do_normalize=UpperCAmelCase_    ,       padding_value=1.0					)

       self.assertEqual(processor.tokenizer.get_vocab()    ,       tokenizer_add_kwargs.get_vocab()					)
       self.assertIsInstance(processor.tokenizer    ,       UpperCAmelCase_					)

       self.assertEqual(processor.feature_extractor.to_json_string()    ,       feature_extractor_add_kwargs.to_json_string()					)
       self.assertIsInstance(processor.feature_extractor    ,       UpperCAmelCase_					)


     def  _A				( self				:		Any					):
       SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			self.get_feature_extractor()
       SCREAMING_SNAKE_CASE				:					List[str]		  =			self.get_tokenizer()

       SCREAMING_SNAKE_CASE				:					str		  =			MusicgenProcessor(tokenizer=UpperCAmelCase_    ,       feature_extractor=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE				:					str		  =			floats_list((3, 1000)					)

       SCREAMING_SNAKE_CASE				:					str		  =			feature_extractor(UpperCAmelCase_    ,       return_tensors="np"					)
       SCREAMING_SNAKE_CASE				:					List[Any]		  =			processor(UpperCAmelCase_    ,       return_tensors="np"					)

       for key in input_feat_extract.keys():
         self.assertAlmostEqual(input_feat_extract[key].sum()    ,       input_processor[key].sum()    ,       delta=1E-2					)


     def  _A				( self				:		Dict					):
       SCREAMING_SNAKE_CASE				:					List[str]		  =			self.get_feature_extractor()
       SCREAMING_SNAKE_CASE				:					Optional[int]		  =			self.get_tokenizer()

       SCREAMING_SNAKE_CASE				:					Optional[int]		  =			MusicgenProcessor(tokenizer=UpperCAmelCase_    ,       feature_extractor=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE				:					Optional[int]		  =			"This is a test string"

       SCREAMING_SNAKE_CASE				:					Tuple		  =			processor(text=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE				:					List[Any]		  =			tokenizer(UpperCAmelCase_					)

       for key in encoded_tok.keys():
         self.assertListEqual(encoded_tok[key]    ,       encoded_processor[key]					)


     def  _A				( self				:		List[Any]					):
       SCREAMING_SNAKE_CASE				:					Any		  =			self.get_feature_extractor()
       SCREAMING_SNAKE_CASE				:					int		  =			self.get_tokenizer()

       SCREAMING_SNAKE_CASE				:					Any		  =			MusicgenProcessor(tokenizer=UpperCAmelCase_    ,       feature_extractor=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE				:					str		  =			[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       SCREAMING_SNAKE_CASE				:					List[Any]		  =			processor.batch_decode(sequences=UpperCAmelCase_					)
       SCREAMING_SNAKE_CASE				:					List[str]		  =			tokenizer.batch_decode(UpperCAmelCase_					)

       self.assertListEqual(UpperCAmelCase_    ,       UpperCAmelCase_					)


     def  _A				( self				:		int					):
       SCREAMING_SNAKE_CASE				:					List[str]		  =			self.get_feature_extractor()
       SCREAMING_SNAKE_CASE				:					List[Any]		  =			self.get_tokenizer()

       SCREAMING_SNAKE_CASE				:					List[str]		  =			MusicgenProcessor(tokenizer=UpperCAmelCase_    ,       feature_extractor=UpperCAmelCase_					)

       self.assertListEqual(
           processor.model_input_names    ,       feature_extractor.model_input_names    ,       msg="`processor` and `feature_extractor` model input names do not match"    ,       )






     def  _A				( self				:		List[str]					):
       SCREAMING_SNAKE_CASE				:					List[Any]		  =			self.get_feature_extractor(padding_side="left"					)
       SCREAMING_SNAKE_CASE				:					str		  =			self.get_tokenizer()

       SCREAMING_SNAKE_CASE				:					List[Any]		  =			MusicgenProcessor(tokenizer=UpperCAmelCase_    ,       feature_extractor=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE				:					List[str]		  =			[floats_list((1, x)					)[0] for x in range(5    ,       20    ,       5					)]
       SCREAMING_SNAKE_CASE				:					List[str]		  =			processor(UpperCAmelCase_					).padding_mask

       SCREAMING_SNAKE_CASE				:					Dict		  =			np.asarray(floats_list((3, 20)					)					)[:, None, :]
       SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			processor.batch_decode(UpperCAmelCase_    ,       padding_mask=UpperCAmelCase_					)

       self.assertIsInstance(UpperCAmelCase_    ,       UpperCAmelCase_					)

       for audio in decoded_audios:
         self.assertIsInstance(UpperCAmelCase_    ,       np.ndarray					)

       self.assertTrue(decoded_audios[0].shape == (1, 10)					)
       self.assertTrue(decoded_audios[1].shape == (1, 15)					)
       self.assertTrue(decoded_audios[2].shape == (1, 20)					)














import csv
import os

import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.csv import CsvDatasetReader, CsvDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def      lowerCamelCase__	(   lowercase  ,   lowercase ):

  """simple docstring"""






  assert isinstance(lowercase  ,   lowercase )
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"  ,   [False, True] )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Dict		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			CsvDatasetReader(lowercase  ,   cache_dir=lowercase  ,   keep_in_memory=lowercase ).read()
  _check_csv_dataset(lowercase  ,   lowercase )


@pytest.mark.parametrize(
    "features"  ,   [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]  ,   )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					Any		  =			tmp_path / "cache"
  # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE				:					Any		  =			features.copy() if features else default_expected_features
  SCREAMING_SNAKE_CASE				:					List[Any]		  =			(
      Features({feature: Value(lowercase ) for feature, dtype in features.items()} ) if features is not None else None
  )
  SCREAMING_SNAKE_CASE				:					Tuple		  =			CsvDatasetReader(lowercase  ,   features=lowercase  ,   cache_dir=lowercase ).read()
  _check_csv_dataset(lowercase  ,   lowercase )


@pytest.mark.parametrize("split"  ,   [None, NamedSplit("train" ), "train", "test"] )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					Dict		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Dict		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE				:					List[Any]		  =			CsvDatasetReader(lowercase  ,   cache_dir=lowercase  ,   split=lowercase ).read()
  _check_csv_dataset(lowercase  ,   lowercase )
  assert dataset.split == split if split else "train"


@pytest.mark.parametrize("path_type"  ,   [str, list] )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  if issubclass(lowercase  ,   lowercase ):
    SCREAMING_SNAKE_CASE				:					Tuple		  =			csv_path
  elif issubclass(lowercase  ,   lowercase ):
    SCREAMING_SNAKE_CASE				:					int		  =			[csv_path]
  SCREAMING_SNAKE_CASE				:					Any		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Optional[int]		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE				:					Tuple		  =			CsvDatasetReader(lowercase  ,   cache_dir=lowercase ).read()
  _check_csv_dataset(lowercase  ,   lowercase )


def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase=("train",) ):

  """simple docstring"""






  assert isinstance(lowercase  ,   lowercase )
  for split in splits:
    SCREAMING_SNAKE_CASE				:					str		  =			dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"  ,   [False, True] )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			CsvDatasetReader({"train": csv_path}  ,   cache_dir=lowercase  ,   keep_in_memory=lowercase ).read()
  _check_csv_datasetdict(lowercase  ,   lowercase )


@pytest.mark.parametrize(
    "features"  ,   [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]  ,   )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					int		  =			tmp_path / "cache"
  # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
  SCREAMING_SNAKE_CASE				:					Any		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			features.copy() if features else default_expected_features
  SCREAMING_SNAKE_CASE				:					List[Any]		  =			(
      Features({feature: Value(lowercase ) for feature, dtype in features.items()} ) if features is not None else None
  )
  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			CsvDatasetReader({"train": csv_path}  ,   features=lowercase  ,   cache_dir=lowercase ).read()
  _check_csv_datasetdict(lowercase  ,   lowercase )


@pytest.mark.parametrize("split"  ,   [None, NamedSplit("train" ), "train", "test"] )
def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  if split:
    SCREAMING_SNAKE_CASE				:					Dict		  =			{split: csv_path}
  else:
    SCREAMING_SNAKE_CASE				:					str		  =			{"train": csv_path, "test": csv_path}
  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Dict		  =			{"col_1": "int64", "col_2": "int64", "col_3": "float64"}
  SCREAMING_SNAKE_CASE				:					Tuple		  =			CsvDatasetReader(lowercase  ,   cache_dir=lowercase ).read()
  _check_csv_datasetdict(lowercase  ,   lowercase  ,   splits=list(path.keys() ) )
  assert all(dataset[split].split == split for split in path.keys() )


def      lowerCamelCase__	(   lowercase ):

  """simple docstring"""






  with open(lowercase  ,   encoding="utf-8" ) as csvfile:
    yield from csv.reader(lowercase )


def      lowerCamelCase__	(   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					List[Any]		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Optional[int]		  =			os.path.join(lowercase  ,   "tmp.csv" )
  SCREAMING_SNAKE_CASE				:					Optional[int]		  =			CsvDatasetReader({"train": csv_path}  ,   cache_dir=lowercase ).read()
  CsvDatasetWriter(dataset["train"]  ,   lowercase  ,   num_proc=1 ).write()

  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			iter_csv_file(lowercase )
  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			iter_csv_file(lowercase )

  for rowa, rowa in zip(lowercase  ,   lowercase ):
    assert rowa == rowa


def      lowerCamelCase__	(   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					List[Any]		  =			os.path.join(lowercase  ,   "tmp.csv" )
  SCREAMING_SNAKE_CASE				:					List[str]		  =			CsvDatasetReader({"train": csv_path}  ,   cache_dir=lowercase ).read()
  CsvDatasetWriter(dataset["train"]  ,   lowercase  ,   num_proc=2 ).write()

  SCREAMING_SNAKE_CASE				:					Dict		  =			iter_csv_file(lowercase )
  SCREAMING_SNAKE_CASE				:					Tuple		  =			iter_csv_file(lowercase )

  for rowa, rowa in zip(lowercase  ,   lowercase ):
    assert rowa == rowa


def      lowerCamelCase__	(   lowercase  ,   lowercase ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			tmp_path / "cache"
  SCREAMING_SNAKE_CASE				:					Union[str, Any]		  =			os.path.join(lowercase  ,   "tmp.csv" )
  SCREAMING_SNAKE_CASE				:					List[str]		  =			CsvDatasetReader({"train": csv_path}  ,   cache_dir=lowercase ).read()
  with pytest.raises(lowercase ):
    CsvDatasetWriter(dataset["train"]  ,   lowercase  ,   num_proc=0 )






