





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


UpperCamelCase_												=	get_tests_dir('fixtures')
UpperCamelCase_												=	get_tests_dir('fixtures/dummy_feature_extractor_config.json')
UpperCamelCase_												=	get_tests_dir('fixtures/dummy-config.json')

class     snake_case_		(   unittest.TestCase							):






 '''simple docstring'''





 def  __UpperCAmelCase						(  self		)	->       List[Any]:
    UpperCAmelCase__						=0

 def  __UpperCAmelCase						(  self		)	->       str:
    UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h"		)
    self.assertIsInstance(lowerCAmelCase_,		lowerCAmelCase_		)

 def  __UpperCAmelCase						(  self		)	->       Tuple:
    UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_		)
    self.assertIsInstance(lowerCAmelCase_,		lowerCAmelCase_		)

 def  __UpperCAmelCase						(  self		)	->       Tuple:
    with tempfile.TemporaryDirectory() as tmpdirname:
       UpperCAmelCase__						=WavaVecaConfig()

       # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_		).to_dict()

       config_dict.pop("feature_extractor_type"		)
       UpperCAmelCase__						=WavaVecaFeatureExtractor(**lowerCAmelCase_		)

       # save in new folder
       model_config.save_pretrained(lowerCAmelCase_		)
       config.save_pretrained(lowerCAmelCase_		)

       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_		)

       # make sure private variable is not incorrectly saved
       UpperCAmelCase__						=json.loads(config.to_json_string()		)
       self.assertTrue("_processor_class" not in dict_as_saved		)

    self.assertIsInstance(lowerCAmelCase_,		lowerCAmelCase_		)

 def  __UpperCAmelCase						(  self		)	->       Any:
    UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_		)
    self.assertIsInstance(lowerCAmelCase_,		lowerCAmelCase_		)

 def  __UpperCAmelCase						(  self		)	->       Any:
    with self.assertRaisesRegex(
        lowerCAmelCase_,		"bert-base is not a local folder and is not a valid model identifier"		):
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained("bert-base"		)

 def  __UpperCAmelCase						(  self		)	->       List[str]:
    with self.assertRaisesRegex(
        lowerCAmelCase_,		R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"		):
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_,		revision="aaaaaa"		)

 def  __UpperCAmelCase						(  self		)	->       Union[str, Any]:
    with self.assertRaisesRegex(
        lowerCAmelCase_,		"hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.",		):
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained("hf-internal-testing/config-no-model"		)

 def  __UpperCAmelCase						(  self		)	->       Tuple:
    # If remote code is not set, we will time out when asking whether to load the model.
    with self.assertRaises(lowerCAmelCase_		):
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(
           "hf-internal-testing/test_dynamic_feature_extractor"		)
    # If remote code is disabled, we can't load this config.
    with self.assertRaises(lowerCAmelCase_		):
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(
           "hf-internal-testing/test_dynamic_feature_extractor",		trust_remote_code=lowerCAmelCase_		)

    UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(
        "hf-internal-testing/test_dynamic_feature_extractor",		trust_remote_code=lowerCAmelCase_		)
    self.assertEqual(feature_extractor.__class__.__name__,		"NewFeatureExtractor"		)

    # Test feature extractor can be reloaded.
    with tempfile.TemporaryDirectory() as tmp_dir:
       feature_extractor.save_pretrained(lowerCAmelCase_		)
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_,		trust_remote_code=lowerCAmelCase_		)
    self.assertEqual(reloaded_feature_extractor.__class__.__name__,		"NewFeatureExtractor"		)

 def  __UpperCAmelCase						(  self		)	->       List[str]:
    try:
       AutoConfig.register("custom",		lowerCAmelCase_		)
       AutoFeatureExtractor.register(lowerCAmelCase_,		lowerCAmelCase_		)
       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(lowerCAmelCase_		):
          AutoFeatureExtractor.register(lowerCAmelCase_,		lowerCAmelCase_		)

       # Now that the config is registered, it can be used as any other config with the auto-API
       UpperCAmelCase__						=CustomFeatureExtractor.from_pretrained(lowerCAmelCase_		)
       with tempfile.TemporaryDirectory() as tmp_dir:
          feature_extractor.save_pretrained(lowerCAmelCase_		)
          UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(lowerCAmelCase_		)
          self.assertIsInstance(lowerCAmelCase_,		lowerCAmelCase_		)

    finally:
       if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
          del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]






 def  __UpperCAmelCase						(  self		)	->       Optional[int]:

    class     snake_case_		(   a_							):






     '''simple docstring'''





     __UpperCamelCase       =      True

    try:
       AutoConfig.register("custom",		lowerCAmelCase_		)
       AutoFeatureExtractor.register(lowerCAmelCase_,		lowerCAmelCase_		)
       # If remote code is not set, the default is to use local
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(
           "hf-internal-testing/test_dynamic_feature_extractor"		)
       self.assertEqual(feature_extractor.__class__.__name__,		"NewFeatureExtractor"		)
       self.assertTrue(feature_extractor.is_local		)

       # If remote code is disabled, we load the local one.
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(
           "hf-internal-testing/test_dynamic_feature_extractor",		trust_remote_code=lowerCAmelCase_		)
       self.assertEqual(feature_extractor.__class__.__name__,		"NewFeatureExtractor"		)
       self.assertTrue(feature_extractor.is_local		)

       # If remote is enabled, we load from the Hub
       UpperCAmelCase__						=AutoFeatureExtractor.from_pretrained(
           "hf-internal-testing/test_dynamic_feature_extractor",		trust_remote_code=lowerCAmelCase_		)
       self.assertEqual(feature_extractor.__class__.__name__,		"NewFeatureExtractor"		)
       self.assertTrue(not hasattr(lowerCAmelCase_,		"is_local"		)		)

    finally:
       if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
          del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







'''simple docstring'''





import re

from filelock import FileLock


try:
					import nltk

					UpperCAmelCase__  			=				True
except (ImportError, ModuleNotFoundError):
					UpperCAmelCase__  			=				False

if NLTK_AVAILABLE:
					with FileLock('''.lock''') as lock:
										nltk.download('''punkt''', quiet=True)






def 						UpperCAmelCase__(				_SCREAMING_SNAKE_CASE					: str ):






			"""simple docstring"""




			re.sub('<n>','',_SCREAMING_SNAKE_CASE )  # remove pegasus newline char
			assert NLTK_AVAILABLE, "nltk must be installed to separate newlines between sentences. (pip install nltk)"
			return "\n".join(nltk.sent_tokenize(_SCREAMING_SNAKE_CASE ) )



