




"""simple docstring"""




import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


lowerCAmelCase							:    Optional[Any]  							=							get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
lowerCAmelCase							:    str  							=							get_tests_dir("""fixtures/vocab.json""")
lowerCAmelCase							:    str  							=							get_tests_dir("""fixtures""")






class 					__magic_name__							( unittest.TestCase       ):






				'''simple docstring'''
				__UpperCamelCase    						=		["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      0





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""				)
							self.assertIsInstance(_a	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowerCamelCase       =      WavaVecaConfig()
										lowerCamelCase       =      AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""				)

										# save in new folder
										model_config.save_pretrained(_a				)
										processor.save_pretrained(_a				)

										lowerCamelCase       =      AutoProcessor.from_pretrained(_a				)

							self.assertIsInstance(_a	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmpdirname:
										# copy relevant files
										copyfile(_a	,				os.path.join(_a	,				_a				)				)
										copyfile(_a	,				os.path.join(_a	,				"""vocab.json"""				)				)

										lowerCamelCase       =      AutoProcessor.from_pretrained(_a				)

							self.assertIsInstance(_a	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowerCamelCase       =      WavaVecaFeatureExtractor()
										lowerCamelCase       =      AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""				)

										lowerCamelCase       =      WavaVecaProcessor(_a	,				_a				)

										# save in new folder
										processor.save_pretrained(_a				)

										# drop `processor_class` in tokenizer
										with open(os.path.join(_a	,				_a				)	,				"""r"""				) as f:
													lowerCamelCase       =      json.load(_a				)
													config_dict.pop("""processor_class"""				)

										with open(os.path.join(_a	,				_a				)	,				"""w"""				) as f:
													f.write(json.dumps(_a				)				)

										lowerCamelCase       =      AutoProcessor.from_pretrained(_a				)

							self.assertIsInstance(_a	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowerCamelCase       =      WavaVecaFeatureExtractor()
										lowerCamelCase       =      AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""				)

										lowerCamelCase       =      WavaVecaProcessor(_a	,				_a				)

										# save in new folder
										processor.save_pretrained(_a				)

										# drop `processor_class` in feature extractor
										with open(os.path.join(_a	,				_a				)	,				"""r"""				) as f:
													lowerCamelCase       =      json.load(_a				)
													config_dict.pop("""processor_class"""				)

										with open(os.path.join(_a	,				_a				)	,				"""w"""				) as f:
													f.write(json.dumps(_a				)				)

										lowerCamelCase       =      AutoProcessor.from_pretrained(_a				)

							self.assertIsInstance(_a	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowerCamelCase       =      WavaVecaConfig(processor_class="""Wav2Vec2Processor"""				)
										model_config.save_pretrained(_a				)
										# copy relevant files
										copyfile(_a	,				os.path.join(_a	,				"""vocab.json"""				)				)
										# create emtpy sample processor
										with open(os.path.join(_a	,				_a				)	,				"""w"""				) as f:
													f.write("""{}"""				)

										lowerCamelCase       =      AutoProcessor.from_pretrained(_a				)

							self.assertIsInstance(_a	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(_a				):
										lowerCamelCase       =      AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""				)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(_a				):
										lowerCamelCase       =      AutoProcessor.from_pretrained(
										    """hf-internal-testing/test_dynamic_processor"""	,				trust_remote_code=_a				)

							lowerCamelCase       =      AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""	,				trust_remote_code=_a				)
							self.assertTrue(processor.special_attribute_present				)
							self.assertEqual(processor.__class__.__name__	,				"""NewProcessor"""				)

							lowerCamelCase       =      processor.feature_extractor
							self.assertTrue(feature_extractor.special_attribute_present				)
							self.assertEqual(feature_extractor.__class__.__name__	,				"""NewFeatureExtractor"""				)

							lowerCamelCase       =      processor.tokenizer
							self.assertTrue(tokenizer.special_attribute_present				)
							if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__	,				"""NewTokenizerFast"""				)

										# Test we can also load the slow version
										lowerCamelCase       =      AutoProcessor.from_pretrained(
										    """hf-internal-testing/test_dynamic_processor"""	,				trust_remote_code=_a	,				use_fast=_a				)
										lowerCamelCase       =      new_processor.tokenizer
										self.assertTrue(new_tokenizer.special_attribute_present				)
										self.assertEqual(new_tokenizer.__class__.__name__	,				"""NewTokenizer"""				)
							else:
										self.assertEqual(tokenizer.__class__.__name__	,				"""NewTokenizer"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							try:
										AutoConfig.register("""custom"""	,				_a				)
										AutoFeatureExtractor.register(_a	,				_a				)
										AutoTokenizer.register(_a	,				slow_tokenizer_class=_a				)
										AutoProcessor.register(_a	,				_a				)
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(_a				):
													AutoProcessor.register(_a	,				_a				)

										# Now that the config is registered, it can be used as any other config with the auto-API
										lowerCamelCase       =      CustomFeatureExtractor.from_pretrained(_a				)

										with tempfile.TemporaryDirectory() as tmp_dir:
													lowerCamelCase       =      os.path.join(_a	,				"""vocab.txt"""				)
													with open(_a	,				"""w"""	,				encoding="""utf-8"""				) as vocab_writer:
																vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
													lowerCamelCase       =      CustomTokenizer(_a				)

										lowerCamelCase       =      CustomProcessor(_a	,				_a				)

										with tempfile.TemporaryDirectory() as tmp_dir:
													processor.save_pretrained(_a				)
													lowerCamelCase       =      AutoProcessor.from_pretrained(_a				)
													self.assertIsInstance(_a	,				_a				)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
										if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""






							class 					__magic_name__							( UpperCAmelCase__       ):






											'''simple docstring'''
											__UpperCamelCase    						=		False






							class 					__magic_name__							( UpperCAmelCase__       ):






											'''simple docstring'''
											__UpperCamelCase    						=		False






							class 					__magic_name__							( UpperCAmelCase__       ):






											'''simple docstring'''
											__UpperCamelCase    						=		"AutoFeatureExtractor"
											__UpperCamelCase    						=		"AutoTokenizer"
											__UpperCamelCase    						=		False

							try:
										AutoConfig.register("""custom"""	,				_a				)
										AutoFeatureExtractor.register(_a	,				_a				)
										AutoTokenizer.register(_a	,				slow_tokenizer_class=_a				)
										AutoProcessor.register(_a	,				_a				)
										# If remote code is not set, the default is to use local classes.
										lowerCamelCase       =      AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""				)
										self.assertEqual(processor.__class__.__name__	,				"""NewProcessor"""				)
										self.assertFalse(processor.special_attribute_present				)
										self.assertFalse(processor.feature_extractor.special_attribute_present				)
										self.assertFalse(processor.tokenizer.special_attribute_present				)

										# If remote code is disabled, we load the local ones.
										lowerCamelCase       =      AutoProcessor.from_pretrained(
										    """hf-internal-testing/test_dynamic_processor"""	,				trust_remote_code=_a				)
										self.assertEqual(processor.__class__.__name__	,				"""NewProcessor"""				)
										self.assertFalse(processor.special_attribute_present				)
										self.assertFalse(processor.feature_extractor.special_attribute_present				)
										self.assertFalse(processor.tokenizer.special_attribute_present				)

										# If remote is enabled, we load from the Hub.
										lowerCamelCase       =      AutoProcessor.from_pretrained(
										    """hf-internal-testing/test_dynamic_processor"""	,				trust_remote_code=_a				)
										self.assertEqual(processor.__class__.__name__	,				"""NewProcessor"""				)
										self.assertTrue(processor.special_attribute_present				)
										self.assertTrue(processor.feature_extractor.special_attribute_present				)
										self.assertTrue(processor.tokenizer.special_attribute_present				)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
										if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-bert"""				)
							self.assertEqual(processor.__class__.__name__	,				"""BertTokenizerFast"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-convnext"""				)
							self.assertEqual(processor.__class__.__name__	,				"""ConvNextImageProcessor"""				)






@is_staging_test
class 					__magic_name__							( unittest.TestCase       ):






				'''simple docstring'''
				__UpperCamelCase    						=		["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





				@classmethod
				def      _lowerCAmelCase       ( cls				):





							"""simple docstring"""
							lowerCamelCase       =      TOKEN
							HfFolder.save_token(_a				)





				@classmethod
				def      _lowerCAmelCase       ( cls				):





							"""simple docstring"""
							try:
										delete_repo(token=cls._token	,				repo_id="""test-processor"""				)
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token	,				repo_id="""valid_org/test-processor-org"""				)
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token	,				repo_id="""test-dynamic-processor"""				)
							except HTTPError:
										pass





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      WavaVecaProcessor.from_pretrained(_a				)
							with tempfile.TemporaryDirectory() as tmp_dir:
										processor.save_pretrained(
										    os.path.join(_a	,				"""test-processor"""				)	,				push_to_hub=_a	,				use_auth_token=self._token				)

										lowerCamelCase       =      WavaVecaProcessor.from_pretrained(f'{USER}/test-processor'				)
										for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(_a	,				getattr(new_processor.feature_extractor	,				_a				)				)
										self.assertDictEqual(new_processor.tokenizer.get_vocab()	,				processor.tokenizer.get_vocab()				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      WavaVecaProcessor.from_pretrained(_a				)

							with tempfile.TemporaryDirectory() as tmp_dir:
										processor.save_pretrained(
										    os.path.join(_a	,				"""test-processor-org"""				)	,				push_to_hub=_a	,				use_auth_token=self._token	,				organization="""valid_org"""	,				)

										lowerCamelCase       =      WavaVecaProcessor.from_pretrained("""valid_org/test-processor-org"""				)
										for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(_a	,				getattr(new_processor.feature_extractor	,				_a				)				)
										self.assertDictEqual(new_processor.tokenizer.get_vocab()	,				processor.tokenizer.get_vocab()				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							CustomFeatureExtractor.register_for_auto_class()
							CustomTokenizer.register_for_auto_class()
							CustomProcessor.register_for_auto_class()

							lowerCamelCase       =      CustomFeatureExtractor.from_pretrained(_a				)

							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCamelCase       =      os.path.join(_a	,				"""vocab.txt"""				)
										with open(_a	,				"""w"""	,				encoding="""utf-8"""				) as vocab_writer:
													vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
										lowerCamelCase       =      CustomTokenizer(_a				)

							lowerCamelCase       =      CustomProcessor(_a	,				_a				)

							with tempfile.TemporaryDirectory() as tmp_dir:
										create_repo(f'{USER}/test-dynamic-processor'	,				token=self._token				)
										lowerCamelCase       =      Repository(_a	,				clone_from=f'{USER}/test-dynamic-processor'	,				token=self._token				)
										processor.save_pretrained(_a				)

										# This has added the proper auto_map field to the feature extractor config
										self.assertDictEqual(
										    processor.feature_extractor.auto_map	,				{
										        """AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor""",
										        """AutoProcessor""": """custom_processing.CustomProcessor""",
										    }	,				)

										# This has added the proper auto_map field to the tokenizer config
										with open(os.path.join(_a	,				"""tokenizer_config.json"""				)				) as f:
													lowerCamelCase       =      json.load(_a				)
										self.assertDictEqual(
										    tokenizer_config["""auto_map"""]	,				{
										        """AutoTokenizer""": ["""custom_tokenization.CustomTokenizer""", None],
										        """AutoProcessor""": """custom_processing.CustomProcessor""",
										    }	,				)

										# The code has been copied from fixtures
										self.assertTrue(os.path.isfile(os.path.join(_a	,				"""custom_feature_extraction.py"""				)				)				)
										self.assertTrue(os.path.isfile(os.path.join(_a	,				"""custom_tokenization.py"""				)				)				)
										self.assertTrue(os.path.isfile(os.path.join(_a	,				"""custom_processing.py"""				)				)				)

										repo.push_to_hub()

							lowerCamelCase       =      AutoProcessor.from_pretrained(f'{USER}/test-dynamic-processor'	,				trust_remote_code=_a				)
							# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
							self.assertEqual(new_processor.__class__.__name__	,				"""CustomProcessor"""				)









"""simple docstring"""




import tempfile
import unittest

from make_student import create_student_by_copying_alternating_layers

from transformers import AutoConfig
from transformers.file_utils import cached_property
from transformers.testing_utils import require_torch


lowerCAmelCase							:    Optional[Any]  							=							"""sshleifer/bart-tiny-random"""
lowerCAmelCase							:    List[Any]  							=							"""patrickvonplaten/t5-tiny-random"""






@require_torch
class 					__magic_name__							( unittest.TestCase       ):






				'''simple docstring'''





				@cached_property
				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return AutoConfig.from_pretrained(_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase   ,     *lowerCamelCase       =      create_student_by_copying_alternating_layers(_a	,				tempfile.mkdtemp()	,				e=1	,				d=1				)
							self.assertEqual(student.config.num_hidden_layers	,				1				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase   ,     *lowerCamelCase       =      create_student_by_copying_alternating_layers(_a	,				tempfile.mkdtemp()	,				e=1	,				d=_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase   ,     *lowerCamelCase       =      create_student_by_copying_alternating_layers(_a	,				tempfile.mkdtemp()	,				e=1	,				d=_a				)
							self.assertEqual(student.config.encoder_layers	,				1				)
							self.assertEqual(student.config.decoder_layers	,				self.teacher_config.encoder_layers				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase   ,     *lowerCamelCase       =      create_student_by_copying_alternating_layers(_a	,				tempfile.mkdtemp()	,				e=1	,				d=1				)
							self.assertEqual(student.config.encoder_layers	,				1				)
							self.assertEqual(student.config.decoder_layers	,				1				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							with self.assertRaises(_a				):
										create_student_by_copying_alternating_layers(_a	,				tempfile.mkdtemp()	,				e=_a	,				d=_a				)



