





import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow







class    __snake_case	:





       '''simple docstring'''







       def 							UpperCAmelCase__			(  self			:   int							,    A			:   Optional[int]							,    A			:   Optional[int]							,    A			:   List[Any]		):
         return None







class    __snake_case	:





       '''simple docstring'''







       def 							UpperCAmelCase__			(  self			:   Optional[Any]							,    A			:   int							,    A			:   Dict							,    A			:   Tuple							,    A			:   Any		):
         return None







class    __snake_case	(							unittest.TestCase      ):





       '''simple docstring'''







       lowerCAmelCase__						=					[
           # (model_name, model_kwargs)
           ("""bert-base-cased""", {}),
           ("""gpt2""", {"""use_cache""": False}),  # We don't support exporting GPT2 past keys anymore
       ]




       @require_tf
       @slow
       def 							UpperCAmelCase__			(  self			:   Tuple		):
         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           self._test_export(A							,    """tf"""							,    12							,    **A		)




       @require_torch
       @slow
       def 							UpperCAmelCase__			(  self			:   Tuple		):
         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           self._test_export(A							,    """pt"""							,    12							,    **A		)




       @require_torch
       @slow
       def 							UpperCAmelCase__			(  self			:   Optional[int]		):
         from transformers import BertModel

         __snake_case:						Union[str, Any]			  =						["""[UNK]""", """[SEP]""", """[CLS]""", """[PAD]""", """[MASK]""", """some""", """other""", """words"""]
         with NamedTemporaryFile(mode="""w+t"""		) as vocab_file:
           vocab_file.write("""\n""".join(A		)		)
           vocab_file.flush()
           __snake_case:						Optional[Any]			  =						BertTokenizerFast(vocab_file.name		)

         with TemporaryDirectory() as bert_save_dir:
           __snake_case:						List[str]			  =						BertModel(BertConfig(vocab_size=len(A		)		)		)
           model.save_pretrained(A		)
           self._test_export(A							,    """pt"""							,    12							,    A		)




       @require_tf
       @slow
       def 							UpperCAmelCase__			(  self			:   Dict		):
         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           __snake_case:						Union[str, Any]			  =						self._test_export(A							,    """tf"""							,    12							,    **A		)
           __snake_case:						Any			  =						quantize(Path(A		)		)

           # Ensure the actual quantized model is not bigger than the original one
           if quantized_path.stat().st_size >= Path(A		).stat().st_size:
             self.fail("""Quantized model is bigger than initial ONNX model"""		)




       @require_torch
       @slow
       def 							UpperCAmelCase__			(  self			:   List[Any]		):
         for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           __snake_case:						Any			  =						self._test_export(A							,    """pt"""							,    12							,    **A		)
           __snake_case:						Optional[int]			  =						quantize(A		)

           # Ensure the actual quantized model is not bigger than the original one
           if quantized_path.stat().st_size >= Path(A		).stat().st_size:
             self.fail("""Quantized model is bigger than initial ONNX model"""		)




       def 							UpperCAmelCase__			(  self			:   List[Any]							,    A			:   List[Any]							,    A			:   Union[str, Any]							,    A			:   Any							,    A			:   Tuple=None							,    **A			:   List[str]		):
         try:
           # Compute path
           with TemporaryDirectory() as tempdir:
             __snake_case:						Union[str, Any]			  =						Path(A		).joinpath("""model.onnx"""		)

           # Remove folder if exists
           if path.parent.exists():
             path.parent.rmdir()

           # Export
           convert(A							,    A							,    A							,    A							,    A							,    **A		)

           return path
         except Exception as e:
           self.fail(A		)




       @require_torch
       @require_tokenizers
       @slow
       def 							UpperCAmelCase__			(  self			:   str		):
         from transformers import BertModel

         __snake_case:						int			  =						BertModel(BertConfig.from_pretrained("""lysandre/tiny-bert-random"""		)		)
         __snake_case:						Optional[int]			  =						BertTokenizerFast.from_pretrained("""lysandre/tiny-bert-random"""		)
         self._test_infer_dynamic_axis(A							,    A							,    """pt"""		)




       @require_tf
       @require_tokenizers
       @slow
       def 							UpperCAmelCase__			(  self			:   Tuple		):
         from transformers import TFBertModel

         __snake_case:						List[Any]			  =						TFBertModel(BertConfig.from_pretrained("""lysandre/tiny-bert-random"""		)		)
         __snake_case:						Dict			  =						BertTokenizerFast.from_pretrained("""lysandre/tiny-bert-random"""		)
         self._test_infer_dynamic_axis(A							,    A							,    """tf"""		)




       def 							UpperCAmelCase__			(  self			:   Any							,    A			:   List[str]							,    A			:   int							,    A			:   Tuple		):
         __snake_case:						List[str]			  =						FeatureExtractionPipeline(A							,    A		)

         __snake_case:						str			  =						["""input_ids""", """token_type_ids""", """attention_mask""", """output_0""", """output_1"""]
         __snake_case:						Dict			  =						infer_shapes(A							,    A		)

         # Assert all variables are present
         self.assertEqual(len(A		)							,    len(A		)		)
         self.assertTrue(all(var_name in shapes for var_name in variable_names		)		)
         self.assertSequenceEqual(variable_names[:3]							,    A		)
         self.assertSequenceEqual(variable_names[3:]							,    A		)

         # Assert inputs are {0: batch, 1: sequence}
         for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
           self.assertDictEqual(shapes[var_name]							,    {0: """batch""", 1: """sequence"""}		)

         # Assert outputs are {0: batch, 1: sequence} and {0: batch}
         self.assertDictEqual(shapes["""output_0"""]							,    {0: """batch""", 1: """sequence"""}		)
         self.assertDictEqual(shapes["""output_1"""]							,    {0: """batch"""}		)




       def 							UpperCAmelCase__			(  self			:   int		):
         __snake_case:						Dict			  =						["""input_ids""", """attention_mask""", """token_type_ids"""]
         __snake_case:						Union[str, Any]			  =						{"""input_ids""": [1, 2, 3, 4], """attention_mask""": [0, 0, 0, 0], """token_type_ids""": [1, 1, 1, 1]}
         __snake_case:						int			  =						ensure_valid_input(FuncContiguousArgs()							,    A							,    A		)

         # Should have exactly the same number of args (all are valid)
         self.assertEqual(len(A		)							,    3		)

         # Should have exactly the same input names
         self.assertEqual(set(A		)							,    set(A		)		)

         # Parameter should be reordered according to their respective place in the function:
         # (input_ids, token_type_ids, attention_mask)
         self.assertEqual(A							,    (tokens["""input_ids"""], tokens["""token_type_ids"""], tokens["""attention_mask"""])		)

         # Generated args are interleaved with another args (for instance parameter "past" in GPT2)
         __snake_case:						Any			  =						ensure_valid_input(FuncNonContiguousArgs()							,    A							,    A		)

         # Should have exactly the one arg (all before the one not provided "some_other_args")
         self.assertEqual(len(A		)							,    1		)
         self.assertEqual(len(A		)							,    1		)

         # Should have only "input_ids"
         self.assertEqual(inputs_args[0]							,    tokens["""input_ids"""]		)
         self.assertEqual(ordered_input_names[0]							,    """input_ids"""		)






       def 							UpperCAmelCase__			(  self			:   Dict		):
         __snake_case:						Any			  =						generate_identified_filename(Path("""/home/something/my_fake_model.onnx"""		)							,    """-test"""		)
         self.assertEqual("""/home/something/my_fake_model-test.onnx"""							,    generated.as_posix()		)








import itertools
import os
import random
import tempfile
import unittest

import numpy as np

from transformers import TvltFeatureExtractor, is_datasets_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
      import torch

if is_datasets_available():
      from datasets import load_dataset

__UpperCAmelCase	:       Any    	=							random.Random()







def       A__							(   SCREAMING_SNAKE_CASE__ , SCREAMING_SNAKE_CASE__=1.0 , SCREAMING_SNAKE_CASE__=None , SCREAMING_SNAKE_CASE__=None)       ->   Any:
  if rng is None:
    __snake_case:						Dict			  =						global_rng

  __snake_case:						str			  =						[]
  for batch_idx in range(shape[0]):
    values.append([])
    for _ in range(shape[1]):
      values[-1].append(rng.random() * scale)

  return values







class    __snake_case	(							unittest.TestCase      ):





       '''simple docstring'''







       def __init__(  self			:   int							,    A			:   List[str]							,    A			:   List[Any]=7							,    A			:   Optional[int]=400							,    A			:   List[Any]=2_000							,    A			:   Dict=2_048							,    A			:   Tuple=128							,    A			:   List[Any]=1							,    A			:   Tuple=512							,    A			:   str=30							,    A			:   Optional[Any]=44_100							,    ):
         __snake_case:						Dict			  =						parent
         __snake_case:						Optional[Any]			  =						batch_size
         __snake_case:						Optional[int]			  =						min_seq_length
         __snake_case:						Optional[Any]			  =						max_seq_length
         __snake_case:						List[str]			  =						(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
         __snake_case:						Any			  =						spectrogram_length
         __snake_case:						Any			  =						feature_size
         __snake_case:						Union[str, Any]			  =						num_audio_channels
         __snake_case:						Any			  =						hop_length
         __snake_case:						List[str]			  =						chunk_length
         __snake_case:						Any			  =						sampling_rate




       def 							UpperCAmelCase__			(  self			:   List[Any]		):
         return {
             "spectrogram_length": self.spectrogram_length,
             "feature_size": self.feature_size,
             "num_audio_channels": self.num_audio_channels,
             "hop_length": self.hop_length,
             "chunk_length": self.chunk_length,
             "sampling_rate": self.sampling_rate,
         }






       def 							UpperCAmelCase__			(  self			:   List[str]							,    A			:   str=False							,    A			:   int=False		):
         def _flatten(A			:   Dict		):
           return list(itertools.chain(*A		)		)

         if equal_length:
           __snake_case:						List[str]			  =						[floats_list((self.max_seq_length, self.feature_size)		) for _ in range(self.batch_size		)]
         else:
           # make sure that inputs increase in size
           __snake_case:						int			  =						[
               floats_list((x, self.feature_size)		)
               for x in range(self.min_seq_length							,    self.max_seq_length							,    self.seq_length_diff		)
           ]
         if numpify:
           __snake_case:						Tuple			  =						[np.asarray(A		) for x in speech_inputs]
         return speech_inputs







@require_torch
@require_torchaudio
class    __snake_case	(							__lowerCamelCase       ,      unittest.TestCase      ):





       '''simple docstring'''







       lowerCAmelCase__						=					TvltFeatureExtractor




       def 							UpperCAmelCase__			(  self			:   Union[str, Any]		):
         __snake_case:						str			  =						TvltFeatureExtractionTester(self		)




       def 							UpperCAmelCase__			(  self			:   int		):
         __snake_case:						Tuple			  =						self.feature_extraction_class(**self.feat_extract_dict		)
         self.assertTrue(hasattr(A							,    """spectrogram_length"""		)		)
         self.assertTrue(hasattr(A							,    """feature_size"""		)		)
         self.assertTrue(hasattr(A							,    """num_audio_channels"""		)		)
         self.assertTrue(hasattr(A							,    """hop_length"""		)		)
         self.assertTrue(hasattr(A							,    """chunk_length"""		)		)
         self.assertTrue(hasattr(A							,    """sampling_rate"""		)		)




       def 							UpperCAmelCase__			(  self			:   Any		):
         __snake_case:						Optional[Any]			  =						self.feature_extraction_class(**self.feat_extract_dict		)

         with tempfile.TemporaryDirectory() as tmpdirname:
           __snake_case:						Tuple			  =						feat_extract_first.save_pretrained(A		)[0]
           check_json_file_has_correct_format(A		)
           __snake_case:						int			  =						self.feature_extraction_class.from_pretrained(A		)

         __snake_case:						List[str]			  =						feat_extract_first.to_dict()
         __snake_case:						str			  =						feat_extract_second.to_dict()
         __snake_case:						List[Any]			  =						dict_first.pop("""mel_filters"""		)
         __snake_case:						str			  =						dict_second.pop("""mel_filters"""		)
         self.assertTrue(np.allclose(A							,    A		)		)
         self.assertEqual(A							,    A		)




       def 							UpperCAmelCase__			(  self			:   Optional[Any]		):
         __snake_case:						str			  =						self.feature_extraction_class(**self.feat_extract_dict		)

         with tempfile.TemporaryDirectory() as tmpdirname:
           __snake_case:						str			  =						os.path.join(A							,    """feat_extract.json"""		)
           feat_extract_first.to_json_file(A		)
           __snake_case:						List[Any]			  =						self.feature_extraction_class.from_json_file(A		)

         __snake_case:						Dict			  =						feat_extract_first.to_dict()
         __snake_case:						Any			  =						feat_extract_second.to_dict()
         __snake_case:						int			  =						dict_first.pop("""mel_filters"""		)
         __snake_case:						int			  =						dict_second.pop("""mel_filters"""		)
         self.assertTrue(np.allclose(A							,    A		)		)
         self.assertEqual(A							,    A		)




       def 							UpperCAmelCase__			(  self			:   Any		):
         # Initialize feature_extractor
         __snake_case:						Union[str, Any]			  =						self.feature_extraction_class(**self.feat_extract_dict		)

         # create three inputs of length 800, 1000, and 1200
         __snake_case:						Dict			  =						[floats_list((1, x)		)[0] for x in range(800							,    1_400							,    200		)]
         __snake_case:						str			  =						[np.asarray(A		) for speech_input in speech_inputs]

         # Test not batched input
         __snake_case:						int			  =						feature_extractor(np_speech_inputs[0]							,    return_tensors="""np"""							,    sampling_rate=44_100		).audio_values

         self.assertTrue(encoded_audios.ndim == 4		)
         self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
         self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
         self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

         # Test batched
         __snake_case:						Optional[int]			  =						feature_extractor(A							,    return_tensors="""np"""							,    sampling_rate=44_100		).audio_values

         self.assertTrue(encoded_audios.ndim == 4		)
         self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
         self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
         self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

         # Test audio masking
         __snake_case:						Union[str, Any]			  =						feature_extractor(
             A							,    return_tensors="""np"""							,    sampling_rate=44_100							,    mask_audio=A		).audio_values

         self.assertTrue(encoded_audios.ndim == 4		)
         self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
         self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
         self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

         # Test 2-D numpy arrays are batched.
         __snake_case:						Any			  =						[floats_list((1, x)		)[0] for x in (800, 800, 800)]
         __snake_case:						Union[str, Any]			  =						np.asarray(A		)
         __snake_case:						List[Any]			  =						feature_extractor(A							,    return_tensors="""np"""							,    sampling_rate=44_100		).audio_values
         self.assertTrue(encoded_audios.ndim == 4		)
         self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
         self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
         self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)




       def 							UpperCAmelCase__			(  self			:   Union[str, Any]							,    A			:   List[str]		):
         __snake_case:						Tuple			  =						load_dataset("""hf-internal-testing/librispeech_asr_dummy"""							,    """clean"""							,    split="""validation"""		)
         # automatic decoding with librispeech
         __snake_case:						List[Any]			  =						ds.sort("""id"""		).select(range(A		)		)[:num_samples]["""audio"""]

         return [x["array"] for x in speech_samples]






       def 							UpperCAmelCase__			(  self			:   Optional[int]		):
         __snake_case:						Dict			  =						self._load_datasamples(1		)
         __snake_case:						Optional[int]			  =						TvltFeatureExtractor()
         __snake_case:						Optional[Any]			  =						feature_extractor(A							,    return_tensors="""pt"""		).audio_values

         self.assertEquals(audio_values.shape							,    (1, 1, 192, 128)		)

         __snake_case:						str			  =						torch.tensor([[-0.3032, -0.2708], [-0.4434, -0.4007]]		)
         self.assertTrue(torch.allclose(audio_values[0, 0, :2, :2]							,    A							,    atol=1E-4		)		)






