






"""simple docstring"""







import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


lowerCAmelCase_	:    Dict            =      '''<<<<<<< This should probably be modified because it mentions: '''

lowerCAmelCase_	:    str            =      '''=======
>>>>>>>
'''

lowerCAmelCase_	:    Union[str, Any]            =      [
    '''TextEncoderConfig''',
    '''ByteTextEncoder''',
    '''SubwordTextEncoder''',
    '''encoder_config''',
    '''maybe_build_from_corpus''',
    '''manual_dir''',
]

lowerCAmelCase_	:    Any            =      [
    # (pattern, replacement)
    # Order is important here for some replacements
    (R'''tfds\.core''', R'''datasets'''),
    (R'''tf\.io\.gfile\.GFile''', R'''open'''),
    (R'''tf\.([\w\d]+)''', R'''datasets.Value(\'\1\')'''),
    (R'''tfds\.features\.Text\(\)''', R'''datasets.Value(\'string\')'''),
    (R'''tfds\.features\.Text\(''', R'''datasets.Value(\'string\'),'''),
    (R'''features\s*=\s*tfds.features.FeaturesDict\(''', R'''features=datasets.Features('''),
    (R'''tfds\.features\.FeaturesDict\(''', R'''dict('''),
    (R'''The TensorFlow Datasets Authors''', R'''The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'''),
    (R'''tfds\.''', R'''datasets.'''),
    (R'''dl_manager\.manual_dir''', R'''self.config.data_dir'''),
    (R'''self\.builder_config''', R'''self.config'''),
]






def       _lowerCAmelCase       (      lowerCAmelCase		):







       '''simple docstring'''
       return ConvertCommand(args.tfds_path	,      args.datasets_directory		)





class   UpperCamelCase_ (       a_  ):



  @staticmethod
  def 		UpperCamelCase_				(      snake_case__				)			->			Tuple:



         """simple docstring"""





         UpperCAmelCase	    =			parser.add_parser(
             """convert""" ,		help="""Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.""" ,		)
         train_parser.add_argument(
             """--tfds_path""" ,		type=snake_case__ ,		required=snake_case__ ,		help="""Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.""" ,		)
         train_parser.add_argument(
             """--datasets_directory""" ,		type=snake_case__ ,		required=snake_case__ ,		help="""Path to the HuggingFace Datasets folder."""				)
         train_parser.set_defaults(func=snake_case__				)



  def __init__(      self ,		snake_case__ ,		snake_case__ ,		*snake_case__				)			->			Any:



         """simple docstring"""





         UpperCAmelCase	    =			get_logger("""datasets-cli/converting"""				)

         UpperCAmelCase	    =			tfds_path
         UpperCAmelCase	    =			datasets_directory



  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         if os.path.isdir(self._tfds_path				):
                UpperCAmelCase	    =			os.path.abspath(self._tfds_path				)
         elif os.path.isfile(self._tfds_path				):
                UpperCAmelCase	    =			os.path.dirname(self._tfds_path				)
         else:
                raise ValueError("""--tfds_path is neither a directory nor a file. Please check path."""				)

         UpperCAmelCase	    =			os.path.abspath(self._datasets_directory				)

         self._logger.info(f'''Converting datasets from {abs_tfds_path} to {abs_datasets_path}'''				)

         UpperCAmelCase	    =			[]
         UpperCAmelCase	    =			[]
         UpperCAmelCase	    =			{}

         if os.path.isdir(self._tfds_path				):
                UpperCAmelCase	    =			os.listdir(snake_case__				)
         else:
                UpperCAmelCase	    =			[os.path.basename(self._tfds_path				)]

         for f_name in file_names:
                self._logger.info(f'''Looking at file {f_name}'''				)
                UpperCAmelCase	    =			os.path.join(snake_case__ ,		snake_case__				)
                UpperCAmelCase	    =			os.path.join(snake_case__ ,		snake_case__				)

                if not os.path.isfile(snake_case__				) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
                       self._logger.info("""Skipping file"""				)
                       continue

                with open(snake_case__ ,		encoding="""utf-8"""				) as f:
                       UpperCAmelCase	    =			f.readlines()

                UpperCAmelCase	    =			[]
                UpperCAmelCase	    =			False
                UpperCAmelCase	    =			False
                UpperCAmelCase	    =			[]
                for line in lines:
                       UpperCAmelCase	    =			line

                       # Convert imports
                       if "import tensorflow.compat.v2 as tf" in out_line:
                              continue
                       elif "@tfds.core" in out_line:
                              continue
                       elif "builder=self" in out_line:
                              continue
                       elif "import tensorflow_datasets.public_api as tfds" in out_line:
                              UpperCAmelCase	    =			"""import datasets\n"""
                       elif "import tensorflow" in out_line:
                              # order is important here
                              UpperCAmelCase	    =			""""""
                              continue
                       elif "from absl import logging" in out_line:
                              UpperCAmelCase	    =			"""from datasets import logging\n"""
                       elif "getLogger" in out_line:
                              UpperCAmelCase	    =			out_line.replace("""getLogger""" ,		"""get_logger"""				)
                       elif any(expression in out_line for expression in TO_HIGHLIGHT				):
                              UpperCAmelCase	    =			True
                              UpperCAmelCase	    =			list(filter(lambda snake_case__				: e in out_line ,		snake_case__				)				)
                              out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(snake_case__				) + """\n"""				)
                              out_lines.append(snake_case__				)
                              out_lines.append(snake_case__				)
                              continue
                       else:
                              for pattern, replacement in TO_CONVERT:
                                     UpperCAmelCase	    =			re.sub(snake_case__ ,		snake_case__ ,		snake_case__				)

                # Take care of saving utilities (to later move them together with main script)
                       if "tensorflow_datasets" in out_line:
                              UpperCAmelCase	    =			re.match(R"""from\stensorflow_datasets.*import\s([^\.\r\n]+)""" ,		snake_case__				)
                              tfds_imports.extend(imp.strip() for imp in match.group(1				).split(""","""				)				)
                              UpperCAmelCase	    =			"""from . import """ + match.group(1				)

                       # Check we have not forget anything
                       if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
                              raise ValueError(f'''Error converting {out_line.strip()}'''				)

                       if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
                              UpperCAmelCase	    =			True
                       out_lines.append(snake_case__				)

                if is_builder or "wmt" in f_name:
                       # We create a new directory for each dataset
                       UpperCAmelCase	    =			f_name.replace(""".py""" ,		""""""				)
                       UpperCAmelCase	    =			os.path.join(snake_case__ ,		snake_case__				)
                       UpperCAmelCase	    =			os.path.join(snake_case__ ,		snake_case__				)
                       os.makedirs(snake_case__ ,		exist_ok=snake_case__				)
                       self._logger.info(f'''Adding directory {output_dir}'''				)
                       imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}				)
                else:
                       # Utilities will be moved at the end
                       utils_files.append(snake_case__				)

                if needs_manual_update:
                       with_manual_update.append(snake_case__				)

                with open(snake_case__ ,		"""w""" ,		encoding="""utf-8"""				) as f:
                       f.writelines(snake_case__				)
                self._logger.info(f'''Converted in {output_file}'''				)

         for utils_file in utils_files:
                try:
                       UpperCAmelCase	    =			os.path.basename(snake_case__				)
                       UpperCAmelCase	    =			imports_to_builder_map[f_name.replace(""".py""" ,		""""""				)]
                       self._logger.info(f'''Moving {dest_folder} to {utils_file}'''				)
                       shutil.copy(snake_case__ ,		snake_case__				)
                except KeyError:
                       self._logger.error(f'''Cannot find destination folder for {utils_file}. Please copy manually.'''				)

         if with_manual_update:
                for file_path in with_manual_update:
                       self._logger.warning(
                           f'''You need to manually update file {file_path} to remove configurations using \'TextEncoderConfig\'.'''				)








"""simple docstring"""







import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import ASTForAudioClassification, ASTModel
 from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
     AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
 )


if is_torchaudio_available():
 import torchaudio

 from transformers import ASTFeatureExtractor





class   UpperCamelCase_ :



  def __init__(      self ,		snake_case__ ,		snake_case__=13 ,		snake_case__=2 ,		snake_case__=24 ,		snake_case__=16 ,		snake_case__=True ,		snake_case__=True ,		snake_case__=32 ,		snake_case__=5 ,		snake_case__=4 ,		snake_case__=37 ,		snake_case__="gelu" ,		snake_case__=0.1 ,		snake_case__=0.1 ,		snake_case__=10 ,		snake_case__=0.02 ,		snake_case__=None ,		snake_case__=2 ,		snake_case__=2 ,		)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			parent
         UpperCAmelCase	    =			batch_size
         UpperCAmelCase	    =			patch_size
         UpperCAmelCase	    =			max_length
         UpperCAmelCase	    =			num_mel_bins
         UpperCAmelCase	    =			is_training
         UpperCAmelCase	    =			use_labels
         UpperCAmelCase	    =			hidden_size
         UpperCAmelCase	    =			num_hidden_layers
         UpperCAmelCase	    =			num_attention_heads
         UpperCAmelCase	    =			intermediate_size
         UpperCAmelCase	    =			hidden_act
         UpperCAmelCase	    =			hidden_dropout_prob
         UpperCAmelCase	    =			attention_probs_dropout_prob
         UpperCAmelCase	    =			type_sequence_label_size
         UpperCAmelCase	    =			initializer_range
         UpperCAmelCase	    =			scope
         UpperCAmelCase	    =			frequency_stride
         UpperCAmelCase	    =			time_stride

         # in AST, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
         UpperCAmelCase	    =			(self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
         UpperCAmelCase	    =			(self.max_length - self.patch_size) // self.time_stride + 1
         UpperCAmelCase	    =			frequency_out_dimension * time_out_dimension
         UpperCAmelCase	    =			num_patches + 2



  def 		UpperCamelCase_				(      self				)			->			Optional[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			floats_tensor([self.batch_size, self.max_length, self.num_mel_bins]				)

         UpperCAmelCase	    =			None
         if self.use_labels:
                UpperCAmelCase	    =			ids_tensor([self.batch_size] ,		self.type_sequence_label_size				)

         UpperCAmelCase	    =			self.get_config()

         return config, input_values, labels



  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         return ASTConfig(
             patch_size=self.patch_size ,		max_length=self.max_length ,		num_mel_bins=self.num_mel_bins ,		hidden_size=self.hidden_size ,		num_hidden_layers=self.num_hidden_layers ,		num_attention_heads=self.num_attention_heads ,		intermediate_size=self.intermediate_size ,		hidden_act=self.hidden_act ,		hidden_dropout_prob=self.hidden_dropout_prob ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob ,		is_decoder=snake_case__ ,		initializer_range=self.initializer_range ,		frequency_stride=self.frequency_stride ,		time_stride=self.time_stride ,		)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__				)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			ASTModel(config=snake_case__				)
         model.to(snake_case__				)
         model.eval()
         UpperCAmelCase	    =			model(snake_case__				)
         self.parent.assertEqual(result.last_hidden_state.shape ,		(self.batch_size, self.seq_length, self.hidden_size)				)



  def 		UpperCamelCase_				(      self				)			->			Optional[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase
         )    ,       (
             UpperCAmelCase
         )    ,       (
             UpperCAmelCase
         )    ,       
         )	    =			config_and_inputs
         UpperCAmelCase	    =			{"""input_values""": input_values}
         return config, inputs_dict





@require_torch
class   UpperCamelCase_ (       a_   ,					a_   ,					unittest.TestCase  ):
  _A       :					Any				=	(
      (
          ASTModel,
          ASTForAudioClassification,
      )
      if is_torch_available()
      else ()
  )
  _A       :					Union[str, Any]				=	(
      {'audio-classification': ASTForAudioClassification, 'feature-extraction': ASTModel}
      if is_torch_available()
      else {}
  )
  _A       :					Any				=	False
  _A       :					Dict				=	False
  _A       :					Optional[Any]				=	False
  _A       :					List[str]				=	False



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__ ,		snake_case__ ,		snake_case__				)			->			Dict:



         """simple docstring"""





         if pipeline_test_casse_name == "AudioClassificationPipelineTests":
                return True

         return False



  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			ASTModelTester(self				)
         UpperCAmelCase	    =			ConfigTester(self ,		config_class=snake_case__ ,		has_text_modality=snake_case__ ,		hidden_size=37				)



  def 		UpperCamelCase_				(      self				)			->			Tuple:



         """simple docstring"""





         self.config_tester.run_common_tests()



  @unittest.skip(reason="""AST does not use inputs_embeds"""				)
  def 		UpperCamelCase_				(      self				)			->			Tuple:



         """simple docstring"""





         pass



  def 		UpperCamelCase_				(      self				)			->			Dict:



         """simple docstring"""





         UpperCAmelCase    ,       UpperCAmelCase	    =			self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
                UpperCAmelCase	    =			model_class(snake_case__				)
                self.assertIsInstance(model.get_input_embeddings() ,		(nn.Module)				)
                UpperCAmelCase	    =			model.get_output_embeddings()
                self.assertTrue(x is None or isinstance(snake_case__ ,		nn.Linear				)				)



  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase    ,       UpperCAmelCase	    =			self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
                UpperCAmelCase	    =			model_class(snake_case__				)
                UpperCAmelCase	    =			inspect.signature(model.forward				)
                # signature.parameters is an OrderedDict => so arg_names order is deterministic
                UpperCAmelCase	    =			[*signature.parameters.keys()]

                UpperCAmelCase	    =			["""input_values"""]
                self.assertListEqual(arg_names[:1] ,		snake_case__				)



  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*snake_case__				)



  @slow
  def 		UpperCamelCase_				(      self				)			->			Union[str, Any]:



         """simple docstring"""





         for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                UpperCAmelCase	    =			ASTModel.from_pretrained(snake_case__				)
                self.assertIsNotNone(snake_case__				)






def       _lowerCAmelCase       (      ):







       '''simple docstring'''
       UpperCAmelCase	    =			hf_hub_download(
           repo_id="""nielsr/audio-spectogram-transformer-checkpoint"""	,      filename="""sample_audio.flac"""	,      repo_type="""dataset"""		)

       UpperCAmelCase    ,       UpperCAmelCase	    =			torchaudio.load(lowerCAmelCase		)

       return audio, sampling_rate





@require_torch
@require_torchaudio
class   UpperCamelCase_ (       unittest.TestCase  ):



  @cached_property
  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         return (
             ASTFeatureExtractor.from_pretrained("""MIT/ast-finetuned-audioset-10-10-0.4593"""				)
             if is_torchaudio_available()
             else None
         )



  @slow
  def 		UpperCamelCase_				(      self				)			->			Optional[int]:



         """simple docstring"""





         UpperCAmelCase	    =			self.default_feature_extractor
         UpperCAmelCase	    =			ASTForAudioClassification.from_pretrained("""MIT/ast-finetuned-audioset-10-10-0.4593"""				).to(snake_case__				)

         UpperCAmelCase	    =			self.default_feature_extractor
         UpperCAmelCase    ,       UpperCAmelCase	    =			prepare_audio()
         UpperCAmelCase	    =			audio.squeeze().numpy()
         UpperCAmelCase	    =			feature_extractor(snake_case__ ,		sampling_rate=snake_case__ ,		return_tensors="""pt"""				).to(snake_case__				)

         # forward pass
         with torch.no_grad():
                UpperCAmelCase	    =			model(**snake_case__				)

         # verify the logits
         UpperCAmelCase	    =			torch.Size((1, 5_27)				)
         self.assertEqual(outputs.logits.shape ,		snake_case__				)

         UpperCAmelCase	    =			torch.tensor([-0.8_760, -7.0_042, -8.6_602]				).to(snake_case__				)

         self.assertTrue(torch.allclose(outputs.logits[0, :3] ,		snake_case__ ,		atol=1e-4				)				)
