'''simple docstring'''


import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


snake_case__					:      Any				      =  get_tests_dir('''fixtures/dummy_feature_extractor_config.json''')
snake_case__					:      List[str]				      =  get_tests_dir('''fixtures/vocab.json''')
snake_case__					:      Dict				      =  get_tests_dir('''fixtures''')






class 					__SCREAMING_SNAKE_CASE						(	unittest.TestCase	):







  '''simple docstring'''






  lowerCamelCase_		:Dict						   =		["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Union[str, Any]       =				0


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Optional[int]       =				AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'						)
       self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmpdirname:
            UpperCAmelCase_							:       Optional[int]       =				WavaVecaConfig()
            UpperCAmelCase_							:       int       =				AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'						)

            # save in new folder
            model_config.save_pretrained(lowerCamelCase_						)
            processor.save_pretrained(lowerCamelCase_						)

            UpperCAmelCase_							:       Any       =				AutoProcessor.from_pretrained(lowerCamelCase_						)

       self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmpdirname:
            # copy relevant files
            copyfile(lowerCamelCase_				,		os.path.join(lowerCamelCase_				,		lowerCamelCase_						)						)
            copyfile(lowerCamelCase_				,		os.path.join(lowerCamelCase_				,		'vocab.json'						)						)

            UpperCAmelCase_							:       Optional[Any]       =				AutoProcessor.from_pretrained(lowerCamelCase_						)

       self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmpdirname:
            UpperCAmelCase_							:       Optional[Any]       =				WavaVecaFeatureExtractor()
            UpperCAmelCase_							:       Optional[int]       =				AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'						)

            UpperCAmelCase_							:       Tuple       =				WavaVecaProcessor(lowerCamelCase_				,		lowerCamelCase_						)

            # save in new folder
            processor.save_pretrained(lowerCamelCase_						)

            # drop `processor_class` in tokenizer
            with open(os.path.join(lowerCamelCase_				,		lowerCamelCase_						)				,		'r'						) as f:
                 UpperCAmelCase_							:       str       =				json.load(lowerCamelCase_						)
                 config_dict.pop('processor_class'						)

            with open(os.path.join(lowerCamelCase_				,		lowerCamelCase_						)				,		'w'						) as f:
                 f.write(json.dumps(lowerCamelCase_						)						)

            UpperCAmelCase_							:       Optional[int]       =				AutoProcessor.from_pretrained(lowerCamelCase_						)

       self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmpdirname:
            UpperCAmelCase_							:       int       =				WavaVecaFeatureExtractor()
            UpperCAmelCase_							:       List[str]       =				AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'						)

            UpperCAmelCase_							:       Tuple       =				WavaVecaProcessor(lowerCamelCase_				,		lowerCamelCase_						)

            # save in new folder
            processor.save_pretrained(lowerCamelCase_						)

            # drop `processor_class` in feature extractor
            with open(os.path.join(lowerCamelCase_				,		lowerCamelCase_						)				,		'r'						) as f:
                 UpperCAmelCase_							:       Optional[int]       =				json.load(lowerCamelCase_						)
                 config_dict.pop('processor_class'						)

            with open(os.path.join(lowerCamelCase_				,		lowerCamelCase_						)				,		'w'						) as f:
                 f.write(json.dumps(lowerCamelCase_						)						)

            UpperCAmelCase_							:       Dict       =				AutoProcessor.from_pretrained(lowerCamelCase_						)

       self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmpdirname:
            UpperCAmelCase_							:       Dict       =				WavaVecaConfig(processor_class='Wav2Vec2Processor'						)
            model_config.save_pretrained(lowerCamelCase_						)
            # copy relevant files
            copyfile(lowerCamelCase_				,		os.path.join(lowerCamelCase_				,		'vocab.json'						)						)
            # create emtpy sample processor
            with open(os.path.join(lowerCamelCase_				,		lowerCamelCase_						)				,		'w'						) as f:
                 f.write('{}'						)

            UpperCAmelCase_							:       Tuple       =				AutoProcessor.from_pretrained(lowerCamelCase_						)

       self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       with self.assertRaises(lowerCamelCase_						):
            UpperCAmelCase_							:       List[str]       =				AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'						)
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(lowerCamelCase_						):
            UpperCAmelCase_							:       List[Any]       =				AutoProcessor.from_pretrained(
                'hf-internal-testing/test_dynamic_processor'				,		trust_remote_code=lowerCamelCase_						)

       UpperCAmelCase_							:       List[str]       =				AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'				,		trust_remote_code=lowerCamelCase_						)
       self.assertTrue(processor.special_attribute_present						)
       self.assertEqual(processor.__class__.__name__				,		'NewProcessor'						)

       UpperCAmelCase_							:       int       =				processor.feature_extractor
       self.assertTrue(feature_extractor.special_attribute_present						)
       self.assertEqual(feature_extractor.__class__.__name__				,		'NewFeatureExtractor'						)

       UpperCAmelCase_							:       Tuple       =				processor.tokenizer
       self.assertTrue(tokenizer.special_attribute_present						)
       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__				,		'NewTokenizerFast'						)

            # Test we can also load the slow version
            UpperCAmelCase_							:       List[Any]       =				AutoProcessor.from_pretrained(
                'hf-internal-testing/test_dynamic_processor'				,		trust_remote_code=lowerCamelCase_				,		use_fast=lowerCamelCase_						)
            UpperCAmelCase_							:       int       =				new_processor.tokenizer
            self.assertTrue(new_tokenizer.special_attribute_present						)
            self.assertEqual(new_tokenizer.__class__.__name__				,		'NewTokenizer'						)
       else:
            self.assertEqual(tokenizer.__class__.__name__				,		'NewTokenizer'						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       try:
            AutoConfig.register('custom'				,		lowerCamelCase_						)
            AutoFeatureExtractor.register(lowerCamelCase_				,		lowerCamelCase_						)
            AutoTokenizer.register(lowerCamelCase_				,		slow_tokenizer_class=lowerCamelCase_						)
            AutoProcessor.register(lowerCamelCase_				,		lowerCamelCase_						)
            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(lowerCamelCase_						):
                 AutoProcessor.register(lowerCamelCase_				,		lowerCamelCase_						)

            # Now that the config is registered, it can be used as any other config with the auto-API
            UpperCAmelCase_							:       Tuple       =				CustomFeatureExtractor.from_pretrained(lowerCamelCase_						)

            with tempfile.TemporaryDirectory() as tmp_dir:
                 UpperCAmelCase_							:       Tuple       =				os.path.join(lowerCamelCase_				,		'vocab.txt'						)
                 with open(lowerCamelCase_				,		'w'				,		encoding='utf-8'						) as vocab_writer:
                      vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
                 UpperCAmelCase_							:       Dict       =				CustomTokenizer(lowerCamelCase_						)

            UpperCAmelCase_							:       Dict       =				CustomProcessor(lowerCamelCase_				,		lowerCamelCase_						)

            with tempfile.TemporaryDirectory() as tmp_dir:
                 processor.save_pretrained(lowerCamelCase_						)
                 UpperCAmelCase_							:       Optional[Any]       =				AutoProcessor.from_pretrained(lowerCamelCase_						)
                 self.assertIsInstance(lowerCamelCase_				,		lowerCamelCase_						)

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                 del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
            if CustomConfig in PROCESSOR_MAPPING._extra_content:
                 del PROCESSOR_MAPPING._extra_content[CustomConfig]


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       class 					__SCREAMING_SNAKE_CASE						(	__snake_case	):







         '''simple docstring'''






         lowerCamelCase_		:List[Any]						   =		False







       class 					__SCREAMING_SNAKE_CASE						(	__snake_case	):







         '''simple docstring'''






         lowerCamelCase_		:Optional[int]						   =		False







       class 					__SCREAMING_SNAKE_CASE						(	__snake_case	):







         '''simple docstring'''






         lowerCamelCase_		:Tuple						   =		"AutoFeatureExtractor"
         lowerCamelCase_		:Any						   =		"AutoTokenizer"
         lowerCamelCase_		:str						   =		False

       try:
            AutoConfig.register('custom'				,		lowerCamelCase_						)
            AutoFeatureExtractor.register(lowerCamelCase_				,		lowerCamelCase_						)
            AutoTokenizer.register(lowerCamelCase_				,		slow_tokenizer_class=lowerCamelCase_						)
            AutoProcessor.register(lowerCamelCase_				,		lowerCamelCase_						)
            # If remote code is not set, the default is to use local classes.
            UpperCAmelCase_							:       Dict       =				AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'						)
            self.assertEqual(processor.__class__.__name__				,		'NewProcessor'						)
            self.assertFalse(processor.special_attribute_present						)
            self.assertFalse(processor.feature_extractor.special_attribute_present						)
            self.assertFalse(processor.tokenizer.special_attribute_present						)

            # If remote code is disabled, we load the local ones.
            UpperCAmelCase_							:       List[Any]       =				AutoProcessor.from_pretrained(
                'hf-internal-testing/test_dynamic_processor'				,		trust_remote_code=lowerCamelCase_						)
            self.assertEqual(processor.__class__.__name__				,		'NewProcessor'						)
            self.assertFalse(processor.special_attribute_present						)
            self.assertFalse(processor.feature_extractor.special_attribute_present						)
            self.assertFalse(processor.tokenizer.special_attribute_present						)

            # If remote is enabled, we load from the Hub.
            UpperCAmelCase_							:       List[Any]       =				AutoProcessor.from_pretrained(
                'hf-internal-testing/test_dynamic_processor'				,		trust_remote_code=lowerCamelCase_						)
            self.assertEqual(processor.__class__.__name__				,		'NewProcessor'						)
            self.assertTrue(processor.special_attribute_present						)
            self.assertTrue(processor.feature_extractor.special_attribute_present						)
            self.assertTrue(processor.tokenizer.special_attribute_present						)

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                 del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
            if CustomConfig in PROCESSOR_MAPPING._extra_content:
                 del PROCESSOR_MAPPING._extra_content[CustomConfig]


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Union[str, Any]       =				AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-bert'						)
       self.assertEqual(processor.__class__.__name__				,		'BertTokenizerFast'						)






  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Tuple       =				AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-convnext'						)
       self.assertEqual(processor.__class__.__name__				,		'ConvNextImageProcessor'						)







@is_staging_test
class 					__SCREAMING_SNAKE_CASE						(	unittest.TestCase	):







  '''simple docstring'''






  lowerCamelCase_		:Any						   =		["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


  @classmethod
  def _UpperCamelCase					(			cls						):




       '''simple docstring'''

       UpperCAmelCase_							:       Dict       =				TOKEN
       HfFolder.save_token(lowerCamelCase_						)


  @classmethod
  def _UpperCamelCase					(			cls						):




       '''simple docstring'''

       try:
            delete_repo(token=cls._token				,		repo_id='test-processor'						)
       except HTTPError:
            pass

       try:
            delete_repo(token=cls._token				,		repo_id='valid_org/test-processor-org'						)
       except HTTPError:
            pass

       try:
            delete_repo(token=cls._token				,		repo_id='test-dynamic-processor'						)
       except HTTPError:
            pass


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Optional[int]       =				WavaVecaProcessor.from_pretrained(lowerCamelCase_						)
       with tempfile.TemporaryDirectory() as tmp_dir:
            processor.save_pretrained(
                os.path.join(lowerCamelCase_				,		'test-processor'						)				,		push_to_hub=lowerCamelCase_				,		use_auth_token=self._token						)

            UpperCAmelCase_							:       Tuple       =				WavaVecaProcessor.from_pretrained(F'''{USER}/test-processor'''						)
            for k, v in processor.feature_extractor.__dict__.items():
                 self.assertEqual(lowerCamelCase_				,		getattr(new_processor.feature_extractor				,		lowerCamelCase_						)						)
            self.assertDictEqual(new_processor.tokenizer.get_vocab()				,		processor.tokenizer.get_vocab()						)


  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       UpperCAmelCase_							:       Union[str, Any]       =				WavaVecaProcessor.from_pretrained(lowerCamelCase_						)

       with tempfile.TemporaryDirectory() as tmp_dir:
            processor.save_pretrained(
                os.path.join(lowerCamelCase_				,		'test-processor-org'						)				,		push_to_hub=lowerCamelCase_				,		use_auth_token=self._token				,		organization='valid_org'				,		)

            UpperCAmelCase_							:       Any       =				WavaVecaProcessor.from_pretrained('valid_org/test-processor-org'						)
            for k, v in processor.feature_extractor.__dict__.items():
                 self.assertEqual(lowerCamelCase_				,		getattr(new_processor.feature_extractor				,		lowerCamelCase_						)						)
            self.assertDictEqual(new_processor.tokenizer.get_vocab()				,		processor.tokenizer.get_vocab()						)






  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       CustomFeatureExtractor.register_for_auto_class()
       CustomTokenizer.register_for_auto_class()
       CustomProcessor.register_for_auto_class()

       UpperCAmelCase_							:       Optional[int]       =				CustomFeatureExtractor.from_pretrained(lowerCamelCase_						)

       with tempfile.TemporaryDirectory() as tmp_dir:
            UpperCAmelCase_							:       str       =				os.path.join(lowerCamelCase_				,		'vocab.txt'						)
            with open(lowerCamelCase_				,		'w'				,		encoding='utf-8'						) as vocab_writer:
                 vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
            UpperCAmelCase_							:       Union[str, Any]       =				CustomTokenizer(lowerCamelCase_						)

       UpperCAmelCase_							:       List[str]       =				CustomProcessor(lowerCamelCase_				,		lowerCamelCase_						)

       with tempfile.TemporaryDirectory() as tmp_dir:
            create_repo(F'''{USER}/test-dynamic-processor'''				,		token=self._token						)
            UpperCAmelCase_							:       str       =				Repository(lowerCamelCase_				,		clone_from=F'''{USER}/test-dynamic-processor'''				,		token=self._token						)
            processor.save_pretrained(lowerCamelCase_						)

            # This has added the proper auto_map field to the feature extractor config
            self.assertDictEqual(
                processor.feature_extractor.auto_map				,		{
                    'AutoFeatureExtractor': 'custom_feature_extraction.CustomFeatureExtractor',
                    'AutoProcessor': 'custom_processing.CustomProcessor',
                }				,		)

            # This has added the proper auto_map field to the tokenizer config
            with open(os.path.join(lowerCamelCase_				,		'tokenizer_config.json'						)						) as f:
                 UpperCAmelCase_							:       Any       =				json.load(lowerCamelCase_						)
            self.assertDictEqual(
                tokenizer_config['auto_map']				,		{
                    'AutoTokenizer': ['custom_tokenization.CustomTokenizer', None],
                    'AutoProcessor': 'custom_processing.CustomProcessor',
                }				,		)

            # The code has been copied from fixtures
            self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase_				,		'custom_feature_extraction.py'						)						)						)
            self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase_				,		'custom_tokenization.py'						)						)						)
            self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase_				,		'custom_processing.py'						)						)						)

            repo.push_to_hub()

       UpperCAmelCase_							:       str       =				AutoProcessor.from_pretrained(F'''{USER}/test-dynamic-processor'''				,		trust_remote_code=lowerCamelCase_						)
       # Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
       self.assertEqual(new_processor.__class__.__name__				,		'CustomProcessor'						)



'''simple docstring'''


snake_case__					:      Optional[Any]				      =  tuple[float, float, float]
snake_case__					:      Tuple				      =  tuple[float, float, float]


def  _lowerCamelCase   (				lowerCamelCase_ :							Pointad     ,					lowerCamelCase_ :							Pointad       ):



     """simple docstring"""





     UpperCAmelCase_							:       Any       =				end_pointa[0] - end_pointa[0]
     UpperCAmelCase_							:       Optional[Any]       =				end_pointa[1] - end_pointa[1]
     UpperCAmelCase_							:       Any       =				end_pointa[2] - end_pointa[2]
     return (x, y, z)


def  _lowerCamelCase   (				lowerCamelCase_ :							Vectorad     ,					lowerCamelCase_ :							Vectorad       ):



     """simple docstring"""





     UpperCAmelCase_							:       Optional[Any]       =				ab[1] * ac[2] - ab[2] * ac[1]  # *i
     UpperCAmelCase_							:       Optional[Any]       =				(ab[0] * ac[2] - ab[2] * ac[0]) * -1  # *j
     UpperCAmelCase_							:       Dict       =				ab[0] * ac[1] - ab[1] * ac[0]  # *k
     return (x, y, z)


def  _lowerCamelCase   (				lowerCamelCase_ :							Vectorad     ,					lowerCamelCase_ :							int       ):



     """simple docstring"""





     return tuple(round(lowerCamelCase_     ,					lowerCamelCase_       ) for x in vector       ) == (0, 0, 0)





def  _lowerCamelCase   (				lowerCamelCase_ :							Pointad     ,					lowerCamelCase_ :							Pointad     ,					lowerCamelCase_ :							Pointad     ,					lowerCamelCase_ :							int = 10       ):



     """simple docstring"""





     UpperCAmelCase_							:       List[str]       =				create_vector(lowerCamelCase_     ,					lowerCamelCase_       )
     UpperCAmelCase_							:       List[Any]       =				create_vector(lowerCamelCase_     ,					lowerCamelCase_       )
     return is_zero_vector(get_ad_vectors_cross(lowerCamelCase_     ,					lowerCamelCase_       )     ,					lowerCamelCase_       )






