

'''simple docstring'''

import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


lowercase__         =      get_tests_dir("fixtures")
lowercase__         =      get_tests_dir("fixtures/dummy_feature_extractor_config.json")
lowercase__         =      get_tests_dir("fixtures/dummy-config.json")




class 	A_			(    unittest.TestCase							):






     '''simple docstring'''



     def        UpperCAmelCase_	(		self			:					Tuple	)						->     Optional[Any]:
      UpperCAmelCase	:     Any						 =			0


     def        UpperCAmelCase_	(		self			:					Optional[int]	)						->     Tuple:
      UpperCAmelCase	:     List[str]						 =			AutoFeatureExtractor.from_pretrained('facebook/wav2vec2-base-960h'	)
      self.assertIsInstance(lowercase_					,      lowercase_	)


     def        UpperCAmelCase_	(		self			:					Dict	)						->     List[Any]:
      UpperCAmelCase	:     Union[str, Any]						 =			AutoFeatureExtractor.from_pretrained(lowercase_	)
      self.assertIsInstance(lowercase_					,      lowercase_	)


     def        UpperCAmelCase_	(		self			:					Union[str, Any]	)						->     int:
      with tempfile.TemporaryDirectory() as tmpdirname:
       UpperCAmelCase	:     Union[str, Any]						 =			WavaVecaConfig()

       # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
       UpperCAmelCase	:     Any						 =			AutoFeatureExtractor.from_pretrained(lowercase_	).to_dict()

       config_dict.pop('feature_extractor_type'	)
       UpperCAmelCase	:     Dict						 =			WavaVecaFeatureExtractor(**lowercase_	)

       # save in new folder
       model_config.save_pretrained(lowercase_	)
       config.save_pretrained(lowercase_	)

       UpperCAmelCase	:     Tuple						 =			AutoFeatureExtractor.from_pretrained(lowercase_	)

       # make sure private variable is not incorrectly saved
       UpperCAmelCase	:     Union[str, Any]						 =			json.loads(config.to_json_string()	)
       self.assertTrue('_processor_class' not in dict_as_saved	)

      self.assertIsInstance(lowercase_					,      lowercase_	)


     def        UpperCAmelCase_	(		self			:					str	)						->     str:
      UpperCAmelCase	:     Optional[Any]						 =			AutoFeatureExtractor.from_pretrained(lowercase_	)
      self.assertIsInstance(lowercase_					,      lowercase_	)


     def        UpperCAmelCase_	(		self			:					Any	)						->     Optional[int]:
      with self.assertRaisesRegex(
          lowercase_					,      'bert-base is not a local folder and is not a valid model identifier'	):
       UpperCAmelCase	:     Union[str, Any]						 =			AutoFeatureExtractor.from_pretrained('bert-base'	)


     def        UpperCAmelCase_	(		self			:					str	)						->     Any:
      with self.assertRaisesRegex(
          lowercase_					,      R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'	):
       UpperCAmelCase	:     str						 =			AutoFeatureExtractor.from_pretrained(lowercase_					,      revision='aaaaaa'	)


     def        UpperCAmelCase_	(		self			:					Dict	)						->     List[str]:
      with self.assertRaisesRegex(
          lowercase_					,      'hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'					,      ):
       UpperCAmelCase	:     List[Any]						 =			AutoFeatureExtractor.from_pretrained('hf-internal-testing/config-no-model'	)


     def        UpperCAmelCase_	(		self			:					int	)						->     List[str]:
      # If remote code is not set, we will time out when asking whether to load the model.
      with self.assertRaises(lowercase_	):
       UpperCAmelCase	:     int						 =			AutoFeatureExtractor.from_pretrained(
           'hf-internal-testing/test_dynamic_feature_extractor'	)
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(lowercase_	):
       UpperCAmelCase	:     Dict						 =			AutoFeatureExtractor.from_pretrained(
           'hf-internal-testing/test_dynamic_feature_extractor'					,      trust_remote_code=lowercase_	)

      UpperCAmelCase	:     List[Any]						 =			AutoFeatureExtractor.from_pretrained(
          'hf-internal-testing/test_dynamic_feature_extractor'					,      trust_remote_code=lowercase_	)
      self.assertEqual(feature_extractor.__class__.__name__					,      'NewFeatureExtractor'	)

      # Test feature extractor can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
       feature_extractor.save_pretrained(lowercase_	)
       UpperCAmelCase	:     List[Any]						 =			AutoFeatureExtractor.from_pretrained(lowercase_					,      trust_remote_code=lowercase_	)
      self.assertEqual(reloaded_feature_extractor.__class__.__name__					,      'NewFeatureExtractor'	)


     def        UpperCAmelCase_	(		self			:					List[str]	)						->     Tuple:
      try:
       AutoConfig.register('custom'					,      lowercase_	)
       AutoFeatureExtractor.register(lowercase_					,      lowercase_	)
       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(lowercase_	):
        AutoFeatureExtractor.register(lowercase_					,      lowercase_	)

       # Now that the config is registered, it can be used as any other config with the auto-API
       UpperCAmelCase	:     Any						 =			CustomFeatureExtractor.from_pretrained(lowercase_	)
       with tempfile.TemporaryDirectory() as tmp_dir:
        feature_extractor.save_pretrained(lowercase_	)
        UpperCAmelCase	:     Dict						 =			AutoFeatureExtractor.from_pretrained(lowercase_	)
        self.assertIsInstance(lowercase_					,      lowercase_	)

      finally:
       if "custom" in CONFIG_MAPPING._extra_content:
        del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
        del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]




     def        UpperCAmelCase_	(		self			:					Any	)						->     int:




      class 	A_			(    _snake_case							):






           '''simple docstring'''



           UpperCAmelCase_						:     Union[str, Any]  		=  True

      try:
       AutoConfig.register('custom'					,      lowercase_	)
       AutoFeatureExtractor.register(lowercase_					,      lowercase_	)
       # If remote code is not set, the default is to use local
       UpperCAmelCase	:     List[Any]						 =			AutoFeatureExtractor.from_pretrained(
           'hf-internal-testing/test_dynamic_feature_extractor'	)
       self.assertEqual(feature_extractor.__class__.__name__					,      'NewFeatureExtractor'	)
       self.assertTrue(feature_extractor.is_local	)

       # If remote code is disabled, we load the local one.
       UpperCAmelCase	:     Union[str, Any]						 =			AutoFeatureExtractor.from_pretrained(
           'hf-internal-testing/test_dynamic_feature_extractor'					,      trust_remote_code=lowercase_	)
       self.assertEqual(feature_extractor.__class__.__name__					,      'NewFeatureExtractor'	)
       self.assertTrue(feature_extractor.is_local	)

       # If remote is enabled, we load from the Hub
       UpperCAmelCase	:     List[str]						 =			AutoFeatureExtractor.from_pretrained(
           'hf-internal-testing/test_dynamic_feature_extractor'					,      trust_remote_code=lowercase_	)
       self.assertEqual(feature_extractor.__class__.__name__					,      'NewFeatureExtractor'	)
       self.assertTrue(not hasattr(lowercase_					,      'is_local'	)	)

      finally:
       if "custom" in CONFIG_MAPPING._extra_content:
        del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
        del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







'''simple docstring'''

import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_     ):
 assert isinstance(UpperCAmelCase_  ,	UpperCAmelCase_     )
 assert dataset.num_rows == 4
 assert dataset.num_columns == 1
 assert dataset.column_names == ["text"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('keep_in_memory'  ,	[False, True]     )
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 UpperCAmelCase	:     int						 =			tmp_path / 'cache'
 UpperCAmelCase	:     List[str]						 =			{'text': 'string'}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  UpperCAmelCase	:     Tuple						 =			TextDatasetReader(UpperCAmelCase_  ,	cache_dir=UpperCAmelCase_  ,	keep_in_memory=UpperCAmelCase_     ).read()
 _check_text_dataset(UpperCAmelCase_  ,	UpperCAmelCase_     )




@pytest.mark.parametrize(
    'features'  ,	[
        None,
        {'text': 'string'},
        {'text': 'int32'},
        {'text': 'float32'},
    ]  ,	)
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 UpperCAmelCase	:     Optional[int]						 =			tmp_path / 'cache'
 UpperCAmelCase	:     List[str]						 =			{'text': 'string'}
 UpperCAmelCase	:     Optional[int]						 =			features.copy() if features else default_expected_features
 UpperCAmelCase	:     int						 =			(
     Features({feature: Value(UpperCAmelCase_     ) for feature, dtype in features.items()}     ) if features is not None else None
 )
 UpperCAmelCase	:     Union[str, Any]						 =			TextDatasetReader(UpperCAmelCase_  ,	features=UpperCAmelCase_  ,	cache_dir=UpperCAmelCase_     ).read()
 _check_text_dataset(UpperCAmelCase_  ,	UpperCAmelCase_     )




@pytest.mark.parametrize('split'  ,	[None, NamedSplit('train'     ), 'train', 'test']     )
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 UpperCAmelCase	:     Optional[int]						 =			tmp_path / 'cache'
 UpperCAmelCase	:     Tuple						 =			{'text': 'string'}
 UpperCAmelCase	:     List[str]						 =			TextDatasetReader(UpperCAmelCase_  ,	cache_dir=UpperCAmelCase_  ,	split=UpperCAmelCase_     ).read()
 _check_text_dataset(UpperCAmelCase_  ,	UpperCAmelCase_     )
 assert dataset.split == split if split else "train"




@pytest.mark.parametrize('path_type'  ,	[str, list]     )
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 if issubclass(UpperCAmelCase_  ,	UpperCAmelCase_     ):
  UpperCAmelCase	:     Tuple						 =			text_path
 elif issubclass(UpperCAmelCase_  ,	UpperCAmelCase_     ):
  UpperCAmelCase	:     Optional[Any]						 =			[text_path]
 UpperCAmelCase	:     List[Any]						 =			tmp_path / 'cache'
 UpperCAmelCase	:     Union[str, Any]						 =			{'text': 'string'}
 UpperCAmelCase	:     List[Any]						 =			TextDatasetReader(UpperCAmelCase_  ,	cache_dir=UpperCAmelCase_     ).read()
 _check_text_dataset(UpperCAmelCase_  ,	UpperCAmelCase_     )




def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_=("train",)     ):
 assert isinstance(UpperCAmelCase_  ,	UpperCAmelCase_     )
 for split in splits:
  UpperCAmelCase	:     Union[str, Any]						 =			dataset_dict[split]
  assert dataset.num_rows == 4
  assert dataset.num_columns == 1
  assert dataset.column_names == ["text"]
  for feature, expected_dtype in expected_features.items():
   assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('keep_in_memory'  ,	[False, True]     )
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 UpperCAmelCase	:     Any						 =			tmp_path / 'cache'
 UpperCAmelCase	:     List[str]						 =			{'text': 'string'}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  UpperCAmelCase	:     int						 =			TextDatasetReader({'train': text_path}  ,	cache_dir=UpperCAmelCase_  ,	keep_in_memory=UpperCAmelCase_     ).read()
 _check_text_datasetdict(UpperCAmelCase_  ,	UpperCAmelCase_     )




@pytest.mark.parametrize(
    'features'  ,	[
        None,
        {'text': 'string'},
        {'text': 'int32'},
        {'text': 'float32'},
    ]  ,	)
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 UpperCAmelCase	:     Union[str, Any]						 =			tmp_path / 'cache'
 # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
 UpperCAmelCase	:     Tuple						 =			{'text': 'string'}
 UpperCAmelCase	:     Union[str, Any]						 =			features.copy() if features else default_expected_features
 UpperCAmelCase	:     int						 =			(
     Features({feature: Value(UpperCAmelCase_     ) for feature, dtype in features.items()}     ) if features is not None else None
 )
 UpperCAmelCase	:     List[Any]						 =			TextDatasetReader({'train': text_path}  ,	features=UpperCAmelCase_  ,	cache_dir=UpperCAmelCase_     ).read()
 _check_text_datasetdict(UpperCAmelCase_  ,	UpperCAmelCase_     )




@pytest.mark.parametrize('split'  ,	[None, NamedSplit('train'     ), 'train', 'test']     )
def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 if split:
  UpperCAmelCase	:     int						 =			{split: text_path}
 else:
  UpperCAmelCase	:     int						 =			'train'
  UpperCAmelCase	:     Any						 =			{'train': text_path, 'test': text_path}
 UpperCAmelCase	:     Dict						 =			tmp_path / 'cache'
 UpperCAmelCase	:     Any						 =			{'text': 'string'}
 UpperCAmelCase	:     List[str]						 =			TextDatasetReader(UpperCAmelCase_  ,	cache_dir=UpperCAmelCase_     ).read()
 _check_text_datasetdict(UpperCAmelCase_  ,	UpperCAmelCase_  ,	splits=list(path.keys()     )     )
 assert all(dataset[split].split == split for split in path.keys()     )




