
import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


_lowercase		:				str      		=random.Random()

if is_torch_available():
		import torch

def      lowerCAmelCase_					(  _lowercase    :       Optional[int]						,       _lowercase    :       Any=1.0						,       _lowercase    :       List[Any]=None						,       _lowercase    :       Tuple=None)							->	Tuple:







			"""simple docstring"""
			if rng is None:
						a__       :						List[Any]							=  global_rng

			a__       :						Optional[Any]							=  []
			for batch_idx in range(shape[0]):
						values.append([])
						for _ in range(shape[1]):
									values[-1].append(rng.random() * scale)

			return values

class        snake_case__    (unittest.TestCase		):







						"""simple docstring"""




						def __init__(       self ,       __lowercase ,       __lowercase=7 ,       __lowercase=4_0_0 ,       __lowercase=2_0_0_0 ,       __lowercase=1 ,       __lowercase=0.0 ,       __lowercase=1_6_0_0_0 ,       __lowercase=True ,       __lowercase=True ,       ) ->				int:
									"""simple docstring"""




									a__       :						List[Any]							=  parent
									a__       :						List[str]							=  batch_size
									a__       :						List[Any]							=  min_seq_length
									a__       :						Optional[int]							=  max_seq_length
									a__       :						str							=  (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
									a__       :						Any							=  feature_size
									a__       :						Optional[Any]							=  padding_value
									a__       :						int							=  sampling_rate
									a__       :						int							=  return_attention_mask
									a__       :						int							=  do_normalize

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				int:
									"""simple docstring"""




									return {
									    "feature_size": self.feature_size,
									    "padding_value": self.padding_value,
									    "sampling_rate": self.sampling_rate,
									    "return_attention_mask": self.return_attention_mask,
									    "do_normalize": self.do_normalize,
									}

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase=False ,       __lowercase=False  ) ->				Union[str, Any]:
									"""simple docstring"""




									def _flatten(__lowercase  ):
												return list(itertools.chain(*__lowercase  )  )

									if equal_length:
												a__       :						Tuple							=  floats_list((self.batch_size, self.max_seq_length)  )
									else:
												# make sure that inputs increase in size
												a__       :						Union[str, Any]							=  [
												    _flatten(floats_list((x, self.feature_size)  )  )
												    for x in range(self.min_seq_length ,       self.max_seq_length ,       self.seq_length_diff  )
												]

									if numpify:
												a__       :						Union[str, Any]							=  [np.asarray(__lowercase  ) for x in speech_inputs]

									return speech_inputs




@require_torch
@require_torchaudio
class        snake_case__    (A__ ,     unittest.TestCase		):







						"""simple docstring"""




						__lowerCAmelCase					:List[Any]      =					ASTFeatureExtractor

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Any:
									"""simple docstring"""




									a__       :						Union[str, Any]							=  ASTFeatureExtractionTester(self  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Dict:
									"""simple docstring"""




									a__       :						Tuple							=  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()  )
									# create three inputs of length 800, 1000, and 1200
									a__       :						Optional[Any]							=  [floats_list((1, x)  )[0] for x in range(8_0_0 ,       1_4_0_0 ,       2_0_0  )]
									a__       :						Union[str, Any]							=  [np.asarray(__lowercase  ) for speech_input in speech_inputs]

									# Test not batched input
									a__       :						Union[str, Any]							=  feat_extract(speech_inputs[0] ,       return_tensors="""np"""  ).input_values
									a__       :						List[str]							=  feat_extract(np_speech_inputs[0] ,       return_tensors="""np"""  ).input_values
									self.assertTrue(np.allclose(__lowercase ,       __lowercase ,       atol=1E-3  )  )

									# Test batched
									a__       :						int							=  feat_extract(__lowercase ,       padding=__lowercase ,       return_tensors="""np"""  ).input_values
									a__       :						Optional[int]							=  feat_extract(__lowercase ,       padding=__lowercase ,       return_tensors="""np"""  ).input_values
									for enc_seq_a, enc_seq_a in zip(__lowercase ,       __lowercase  ):
												self.assertTrue(np.allclose(__lowercase ,       __lowercase ,       atol=1E-3  )  )

									# Test 2-D numpy arrays are batched.
									a__       :						Union[str, Any]							=  [floats_list((1, x)  )[0] for x in (8_0_0, 8_0_0, 8_0_0)]
									a__       :						int							=  np.asarray(__lowercase  )
									a__       :						Optional[Any]							=  feat_extract(__lowercase ,       return_tensors="""np"""  ).input_values
									a__       :						Optional[int]							=  feat_extract(__lowercase ,       return_tensors="""np"""  ).input_values
									for enc_seq_a, enc_seq_a in zip(__lowercase ,       __lowercase  ):
												self.assertTrue(np.allclose(__lowercase ,       __lowercase ,       atol=1E-3  )  )

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									import torch

									a__       :						int							=  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()  )
									a__       :						Any							=  np.random.rand(1_0_0  ).astype(np.floataa  )
									a__       :						Any							=  np_speech_inputs.tolist()

									for inputs in [py_speech_inputs, np_speech_inputs]:
												a__       :						int							=  feature_extractor.pad([{"""input_values""": inputs}] ,       return_tensors="""np"""  )
												self.assertTrue(np_processed.input_values.dtype == np.floataa  )
												a__       :						List[Any]							=  feature_extractor.pad([{"""input_values""": inputs}] ,       return_tensors="""pt"""  )
												self.assertTrue(pt_processed.input_values.dtype == torch.floataa  )

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				Optional[int]:
									"""simple docstring"""




									from datasets import load_dataset

									a__       :						List[Any]							=  load_dataset("""hf-internal-testing/librispeech_asr_dummy""" ,       """clean""" ,       split="""validation"""  )
									# automatic decoding with librispeech
									a__       :						Union[str, Any]							=  ds.sort("""id"""  ).select(range(__lowercase  )  )[:num_samples]["""audio"""]

									return [x["array"] for x in speech_samples]

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									a__       :						Optional[int]							=  torch.tensor(
									    [-0.9_8_9_4, -1.2_7_7_6, -0.9_0_6_6, -1.2_7_7_6, -0.9_3_4_9, -1.2_6_0_9, -1.0_3_8_6, -1.2_7_7_6,
									     -1.1_5_6_1, -1.2_7_7_6, -1.2_0_5_2, -1.2_7_2_3, -1.2_1_9_0, -1.2_1_3_2, -1.2_7_7_6, -1.1_1_3_3,
									     -1.1_9_5_3, -1.1_3_4_3, -1.1_5_8_4, -1.2_2_0_3, -1.1_7_7_0, -1.2_4_7_4, -1.2_3_8_1, -1.1_9_3_6,
									     -0.9_2_7_0, -0.8_3_1_7, -0.8_0_4_9, -0.7_7_0_6, -0.7_5_6_5, -0.7_8_6_9]  )
									# fmt: on

									a__       :						Any							=  self._load_datasamples(1  )
									a__       :						Tuple							=  ASTFeatureExtractor()
									a__       :						int							=  feature_extractor(__lowercase ,       return_tensors="""pt"""  ).input_values
									self.assertEquals(input_values.shape ,       (1, 1_0_2_4, 1_2_8)  )
									self.assertTrue(torch.allclose(input_values[0, 0, :3_0] ,       __lowercase ,       atol=1E-4  )  )









import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowercase		:				Union[str, Any]      		=data_utils.TransfoXLTokenizer
_lowercase		:				Tuple      		=data_utils.TransfoXLCorpus
_lowercase		:				Optional[int]      		=data_utils
_lowercase		:				int      		=data_utils

def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       List[Any]						,       _lowercase    :       Optional[Any]						,       _lowercase    :       Optional[Any])							->	List[Any]:







			"""simple docstring"""
			if transfo_xl_dataset_file:
						# Convert a pre-processed corpus (see original TensorFlow repo)
						with open(_lowercase						,       """rb""") as fp:
									a__       :						Optional[Any]							=  pickle.load(_lowercase						,       encoding="""latin1""")
						# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
						a__       :						Dict							=  pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
						print(F'''Save vocabulary to {pytorch_vocab_dump_path}''')
						a__       :						List[str]							=  corpus.vocab.__dict__
						torch.save(_lowercase						,       _lowercase)

						a__       :						Optional[int]							=  corpus.__dict__
						corpus_dict_no_vocab.pop("""vocab"""						,       _lowercase)
						a__       :						Tuple							=  pytorch_dump_folder_path + """/""" + CORPUS_NAME
						print(F'''Save dataset to {pytorch_dataset_dump_path}''')
						torch.save(_lowercase						,       _lowercase)

			if tf_checkpoint_path:
						# Convert a pre-trained TensorFlow model
						a__       :						Optional[Any]							=  os.path.abspath(_lowercase)
						a__       :						str							=  os.path.abspath(_lowercase)

						print(F'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.''')
						# Initialise PyTorch model
						if transfo_xl_config_file == "":
									a__       :						List[str]							=  TransfoXLConfig()
						else:
									a__       :						Any							=  TransfoXLConfig.from_json_file(_lowercase)
						print(F'''Building PyTorch model from configuration: {config}''')
						a__       :						Any							=  TransfoXLLMHeadModel(_lowercase)

						a__       :						Optional[Any]							=  load_tf_weights_in_transfo_xl(_lowercase						,       _lowercase						,       _lowercase)
						# Save pytorch-model
						a__       :						Tuple							=  os.path.join(_lowercase						,       _lowercase)
						a__       :						Optional[int]							=  os.path.join(_lowercase						,       _lowercase)
						print(F'''Save PyTorch model to {os.path.abspath(_lowercase)}''')
						torch.save(model.state_dict()						,       _lowercase)
						print(F'''Save configuration file to {os.path.abspath(_lowercase)}''')
						with open(_lowercase						,       """w"""						,       encoding="""utf-8""") as f:
									f.write(config.to_json_string())


if __name__ == "__main__":
		_lowercase		:				Tuple      		=argparse.ArgumentParser()
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default=None,
		    type=str,
		    required=True,
		    help="Path to the folder to store the PyTorch model or dataset/vocab.",
		)
		parser.add_argument(
		    "--tf_checkpoint_path",
		    default="",
		    type=str,
		    help="An optional path to a TensorFlow checkpoint path to be converted.",
		)
		parser.add_argument(
		    "--transfo_xl_config_file",
		    default="",
		    type=str,
		    help=(
		        "An optional config json file corresponding to the pre-trained BERT model. \n"
		        "This specifies the model architecture."
		    ),
		)
		parser.add_argument(
		    "--transfo_xl_dataset_file",
		    default="",
		    type=str,
		    help="An optional dataset file to be converted in a vocabulary.",
		)
		_lowercase		:				Dict      		=parser.parse_args()
		convert_transfo_xl_checkpoint_to_pytorch(
		    args.tf_checkpoint_path,
		    args.transfo_xl_config_file,
		    args.pytorch_dump_folder_path,
		    args.transfo_xl_dataset_file,
		)




