text: import os
  import numpy as np
  import matplotlib
  import nlpaug.augmenter.audio as naa
  import tensorflow as tf
  from keras.layers import Add, Lambda, Concatenate, SpatialDropout1D
  
  import keras
  from keras.layers import Input, Activation, Dense, Conv1D, Dropout, BatchNormalization
  from keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard
  from keras.models import load_model, Model
  from keras import optimizers
  from keras.layers.advanced_activations import LeakyReLU
  from  keras import backend as K
  from sklearn.metrics import f1_score, accuracy_score
  from tqdm import tqdm
  import matplotlib.pyplot as plt
  import librosa
  # import librosa.filters
  from joblib import Parallel, delayed
  import multiprocessing
  from .constants import model_params, base_data_path
  from scipy import signal
  from scipy.io import wavfile
  from skopt import gp_minimize
  from skopt.space import Real
  from functools import partial
  from pydub import AudioSegment
  from keras.utils import multi_gpu_model
  
  from .constants import *
  
  #Set a random seed for numpy for reproducibility
  np.random.seed(42)
  
  if os.environ.get('DISPLAY','') == '':
      print('no display found. Using non-interactive Agg backend')
      matplotlib.use('Agg')
  
  
  try:
      import foundations
  except Exception as e:
      print(e)
  
  
  
  def load_wav(path, sr):
  	return librosa.core.load(path, sr=sr)[0]
  
  def save_wav(wav, path, sr):
  	wav *= 32767 / max(0.01, np.max(np.abs(wav)))
  	#proposed by @dsmiller
  	wavfile.write(path, sr, wav.astype(np.int16))
  
  def save_wavenet_wav(wav, path, sr, inv_preemphasize, k):
  	# wav = inv_preemphasis(wav, k, inv_preemphasize)
  	wav *= 32767 / max(0.01, np.max(np.abs(wav)))
  	wavfile.write(path, sr, wav.astype(np.int16))
  
  def preemphasis(wav, k, preemphasize=True):
  	if preemphasize:
  		return signal.lfilter([1, -k], [1], wav)
  	return wav
  
  def inv_preemphasis(wav, k, inv_preemphasize=True):
  	if inv_preemphasize:
  		return signal.lfilter([1], [1, -k], wav)
  	return wav
  
  #From https://github.com/r9y9/wavenet_vocoder/blob/master/audio.py
  def start_and_end_indices(quantized, silence_threshold=2):
  	for start in range(quantized.size):
  		if abs(quantized[start] - 127) > silence_threshold:
  			break
  	for end in range(quantized.size - 1, 1, -1):
  		if abs(quantized[end] - 127) > silence_threshold:
  			break
  
  	assert abs(quantized[start] - 127) > silence_threshold
  	assert abs(quantized[end] - 127) > silence_threshold
  
  	return start, end
  
  def trim_silence(wav, hparams):
  	'''Trim leading and trailing silence
  	Useful for M-AILABS dataset if we choose to trim the extra 0.5 silence at beginning and end.
  	'''
  	#Thanks @begeekmyfriend and @lautjy for pointing out the params contradiction. These params are separate and tunable per dataset.
  	return librosa.effects.trim(wav, top_db= hparams.trim_top_db, frame_length=hparams.trim_fft_size, hop_length=hparams.trim_hop_size)[0]
  
  def get_hop_size(hparams):
  	hop_size = hparams.hop_size
  	if hop_size is None:
  		assert hparams.frame_shift_ms is not None
  		hop_size = int(hparams.frame_shift_ms / 1000 * hparams.sample_rate)
  	return hop_size
  
  def linearspectrogram(wav, hparams):
  	# D = _stft(preemphasis(wav, hparams.preemphasis, hparams.preemphasize), hparams)
  	D = _stft(wav, hparams)
  	S = _amp_to_db(np.abs(D)**hparams.magnitude_power, hparams) - hparams.ref_level_db
  
  	if hparams.signal_normalization:
  		return _normalize(S, hparams)
  	return S
  
  def melspectrogram(wav, hparams):
  	# D = _stft(preemphasis(wav, hparams.preemphasis, hparams.preemphasize), hparams)
  	D = _stft(wav, hparams)
  	S = _amp_to_db(_linear_to_mel(np.abs(D)**hparams.magnitude_power, hparams), hparams) - hparams.ref_level_db
  
  	if hparams.signal_normalization:
  		return _normalize(S, hparams)
  	return S
  
  def inv_linear_spectrogram(linear_spectrogram, hparams):
  	'''Converts linear spectrogram to waveform using librosa'''
  	if hparams.signal_normalization:
  		D = _denormalize(linear_spectrogram, hparams)
  	else:
  		D = linear_spectrogram
  
  	S = _db_to_amp(D + hparams.ref_level_db)**(1/hparams.magnitude_power) #Convert back to linear
  
  	if hparams.use_lws:
  		processor = _lws_processor(hparams)
  		D = processor.run_lws(S.astype(np.float64).T ** hparams.power)
  		y = processor.istft(D).astype(np.float32)
  		return inv_preemphasis(y, hparams.preemphasis, hparams.preemphasize)
  	else:
  		return inv_preemphasis(_griffin_lim(S ** hparams.power, hparams), hparams.preemphasis, hparams.preemphasize)
  
  
  def inv_mel_spectrogram(mel_spectrogram, hparams):
  	'''Converts mel spectrogram to waveform using librosa'''
  	if hparams.signal_normalization:
  		D = _denormalize(mel_spectrogram, hparams)
  	else:
  		D = mel_spectrogram
  
  	S = _mel_to_linear(_db_to_amp(D + hparams.ref_level_db)**(1/hparams.magnitude_power), hparams)  # Convert back to linear
  
  	if hparams.use_lws:
  		processor = _lws_processor(hparams)
  		D = processor.run_lws(S.astype(np.float64).T ** hparams.power)
  		y = processor.istft(D).astype(np.float32)
  		return inv_preemphasis(y, hparams.preemphasis, hparams.preemphasize)
  	else:
  		return inv_preemphasis(_griffin_lim(S ** hparams.power, hparams), hparams.preemphasis, hparams.preemphasize)
  
  ###########################################################################################
  # tensorflow Griffin-Lim
  # Thanks to @begeekmyfriend: https://github.com/begeekmyfriend/Tacotron-2/blob/mandarin-new/datasets/audio.py
  
  def inv_linear_spectrogram_tensorflow(spectrogram, hparams):
  	'''Builds computational graph to convert spectrogram to waveform using TensorFlow.
  	Unlike inv_spectrogram, this does NOT invert the preemphasis. The caller should call
  	inv_preemphasis on the output after running the graph.
  	'''
  	if hparams.signal_normalization:
  		D = _denormalize_tensorflow(spectrogram, hparams)
  	else:
  		D = linear_spectrogram
  
  	S = tf.pow(_db_to_amp_tensorflow(D + hparams.ref_level_db), (1/hparams.magnitude_power))
  	return _griffin_lim_tensorflow(tf.pow(S, hparams.power), hparams)
  
  def inv_mel_spectrogram_tensorflow(mel_spectrogram, hparams):
  	'''Builds computational graph to convert mel spectrogram to waveform using TensorFlow.
  	Unlike inv_mel_spectrogram, this does NOT invert the preemphasis. The caller should call
  	inv_preemphasis on the output after running the graph.
  	'''
  	if hparams.signal_normalization:
  		D = _denormalize_tensorflow(mel_spectrogram, hparams)
  	else:
  		D = mel_spectrogram
  
  	S = tf.pow(_db_to_amp_tensorflow(D + hparams.ref_level_db), (1/hparams.magnitude_power))
  	S = _mel_to_linear_tensorflow(S, hparams)  # Convert back to linear
  	return _griffin_lim_tensorflow(tf.pow(S, hparams.power), hparams)
  
  ###########################################################################################
  
  def _lws_processor(hparams):
  	import lws
  	return lws.lws(hparams.n_fft, get_hop_size(hparams), fftsize=hparams.win_size, mode="speech")
  
  def _griffin_lim(S, hparams):
  	'''librosa implementation of Griffin-Lim
  	Based on https://github.com/librosa/librosa/issues/434
  	'''
  	angles = np.exp(2j * np.pi * np.random.rand(*S.shape))
  	S_complex = np.abs(S).astype(np.complex)
  	y = _istft(S_complex * angles, hparams)
  	for i in range(hparams.griffin_lim_iters):
  		angles = np.exp(1j * np.angle(_stft(y, hparams)))
  		y = _istft(S_complex * angles, hparams)
  	return y
  
  def _griffin_lim_tensorflow(S, hparams):
  	'''TensorFlow implementation of Griffin-Lim
  	Based on https://github.com/Kyubyong/tensorflow-exercises/blob/master/Audio_Processing.ipynb
  	'''
  	with tf.variable_scope('griffinlim'):
  		# TensorFlow's stft and istft operate on a batch of spectrograms; create batch of size 1
  		S = tf.expand_dims(S, 0)
  		S_complex = tf.identity(tf.cast(S, dtype=tf.complex64))
  		y = tf.contrib.signal.inverse_stft(S_complex, hparams.win_size, get_hop_size(hparams), hparams.n_fft)
  		for i in range(hparams.griffin_lim_iters):
  			est = tf.contrib.signal.stft(y, hparams.win_size, get_hop_size(hparams), hparams.n_fft)
  			angles = est / tf.cast(tf.maximum(1e-8, tf.abs(est)), tf.complex64)
  			y = tf.contrib.signal.inverse_stft(S_complex * angles, hparams.win_size, get_hop_size(hparams), hparams.n_fft)
  	return tf.squeeze(y, 0)
  
  def _stft(y, hparams):
  	if hparams.use_lws:
  		return _lws_processor(hparams).stft(y).T
  	else:
  		return librosa.stft(y=y, n_fft=hparams.n_fft, hop_length=get_hop_size(hparams), win_length=hparams.win_size, pad_mode='constant')
  
  def _istft(y, hparams):
  	return librosa.istft(y, hop_length=get_hop_size(hparams), win_length=hparams.win_size)
  
  ##########################################################
  #Those are only correct when using lws!!! (This was messing with Wavenet quality for a long time!)
  def num_frames(length, fsize, fshift):
  	"""Compute number of time frames of spectrogram
  	"""
  	pad = (fsize - fshift)
  	if length % fshift == 0:
  		M = (length + pad * 2 - fsize) // fshift + 1
  	else:
  		M = (length + pad * 2 - fsize) // fshift + 2
  	return M
  
  
  def pad_lr(x, fsize, fshift):
  	"""Compute left and right padding
  	"""
  	M = num_frames(len(x), fsize, fshift)
  	pad = (fsize - fshift)
  	T = len(x) + 2 * pad
  	r = (M - 1) * fshift + fsize - T
  	return pad, pad + r
  ##########################################################
  #Librosa correct padding
  def librosa_pad_lr(x, fsize, fshift, pad_sides=1):
  	'''compute right padding (final frame) or both sides padding (first and final frames)
  	'''
  	assert pad_sides in (1, 2)
  	# return int(fsize // 2)
  	pad = (x.shape[0] // fshift + 1) * fshift - x.shape[0]
  	if pad_sides == 1:
  		return 0, pad
  	else:
  		return pad // 2, pad // 2 + pad % 2
  
  # Conversions
  _mel_basis = None
  _inv_mel_basis = None
  
  def _linear_to_mel(spectogram, hparams):
  	global _mel_basis
  	if _mel_basis is None:
  		_mel_basis = _build_mel_basis(hparams)
  	return np.dot(_mel_basis, spectogram)
  
  def _mel_to_linear(mel_spectrogram, hparams):
  	global _inv_mel_basis
  	if _inv_mel_basis is None:
  		_inv_mel_basis = np.linalg.pinv(_build_mel_basis(hparams))
  	return np.maximum(1e-10, np.dot(_inv_mel_basis, mel_spectrogram))
  
  def _mel_to_linear_tensorflow(mel_spectrogram, hparams):
  	global _inv_mel_basis
  	if _inv_mel_basis is None:
  		_inv_mel_basis = np.linalg.pinv(_build_mel_basis(hparams))
  	return tf.transpose(tf.maximum(1e-10, tf.matmul(tf.cast(_inv_mel_basis, tf.float32), tf.transpose(mel_spectrogram, [1, 0]))), [1, 0])
  
  def _build_mel_basis(hparams):
  	assert hparams.fmax <= hparams.sample_rate // 2
  	return librosa.filters.mel(hparams.sample_rate, hparams.n_fft, n_mels=hparams.num_mels,
  							   fmin=hparams.fmin, fmax=hparams.fmax)
  
  def _amp_to_db(x, hparams):
  	min_level = np.exp(hparams.min_level_db / 20 * np.log(10))
  	return 20 * np.log10(np.maximum(min_level, x))
  
  def _db_to_amp(x):
  	return np.power(10.0, (x) * 0.05)
  
  def _db_to_amp_tensorflow(x):
  	return tf.pow(tf.ones(tf.shape(x)) * 10.0, x * 0.05)
  
  def _normalize(S, hparams):
  	if hparams.allow_clipping_in_normalization:
  		if hparams.symmetric_mels:
  			return np.clip((2 * hparams.max_abs_value) * ((S - hparams.min_level_db) / (-hparams.min_level_db)) - hparams.max_abs_value,
  			 -hparams.max_abs_value, hparams.max_abs_value)
  		else:
  			return np.clip(hparams.max_abs_value * ((S - hparams.min_level_db) / (-hparams.min_level_db)), 0, hparams.max_abs_value)
  
  	assert S.max() <= 0 and S.min() - hparams.min_level_db >= 0
  	if hparams.symmetric_mels:
  		return (2 * hparams.max_abs_value) * ((S - hparams.min_level_db) / (-hparams.min_level_db)) - hparams.max_abs_value
  	else:
  		return hparams.max_abs_value * ((S - hparams.min_level_db) / (-hparams.min_level_db))
  
  def _denormalize(D, hparams):
  	if hparams.allow_clipping_in_normalization:
  		if hparams.symmetric_mels:
  			return (((np.clip(D, -hparams.max_abs_value,
  				hparams.max_abs_value) + hparams.max_abs_value) * -hparams.min_level_db / (2 * hparams.max_abs_value))
  				+ hparams.min_level_db)
  		else:
  			return ((np.clip(D, 0, hparams.max_abs_value) * -hparams.min_level_db / hparams.max_abs_value) + hparams.min_level_db)
  
  	if hparams.symmetric_mels:
  		return (((D + hparams.max_abs_value) * -hparams.min_level_db / (2 * hparams.max_abs_value)) + hparams.min_level_db)
  	else:
  		return ((D * -hparams.min_level_db / hparams.max_abs_value) + hparams.min_level_db)
  
  def _denormalize_tensorflow(D, hparams):
  	if hparams.allow_clipping_in_normalization:
  		if hparams.symmetric_mels:
  			return (((tf.clip_by_value(D, -hparams.max_abs_value,
  				hparams.max_abs_value) + hparams.max_abs_value) * -hparams.min_level_db / (2 * hparams.max_abs_value))
  				+ hparams.min_level_db)
  		else:
  			return ((tf.clip_by_value(D, 0, hparams.max_abs_value) * -hparams.min_level_db / hparams.max_abs_value) + hparams.min_level_db)
  
  	if hparams.symmetric_mels:
  		return (((D + hparams.max_abs_value) * -hparams.min_level_db / (2 * hparams.max_abs_value)) + hparams.min_level_db)
  	else:
  		return ((D * -hparams.min_level_db / hparams.max_abs_value) + hparams.min_level_db)
  
  
  
  
  # given a path, return list of all files in directory
  def get_list_of_wav_files(file_path):
      files = os.listdir(file_path)
      absolute_given_dir = os.path.abspath(file_path)
      absolute_files = list(map(lambda file_path: os.path.join(absolute_given_dir, file_path), files))
      return absolute_files
  
  
  def convert_to_flac(dir_path):
      for file_path in os.listdir(dir_path):
          if file_path.split('.')[-1] != "flac":
              read_file = AudioSegment.from_file(os.path.join(dir_path,file_path), file_path.split('.')[-1])
              os.remove(os.path.join(dir_path,file_path))
              base_name = file_path.split('.')[:-1]
              # read_file = read_file.set_channels(8)
              # base_name = ".".join(base_name)
              read_file.export(os.path.join(dir_path,f"{base_name[0]}.flac"), format="flac")
  
  def get_target(file_path):
      if '/real/' in file_path:
          return 'real'
      elif '/fake/' in file_path:
          return 'fake'
  
  
  def save_wav_to_npy(output_file, spectrogram):
      np.save(output_file, spectrogram)
  
  
  def wav_to_mel(input_file, output_path):
      y, sr = librosa.load(input_file)
      filename = os.path.basename(input_file)
      target = get_target(input_file)
  
      output_file = '{}{}-{}'.format(output_path, filename.split('.')[0], target)
  
      mel_spectrogram_of_audio = librosa.feature.melspectrogram(y=y, sr=sr).T
      save_wav_to_npy(output_file, mel_spectrogram_of_audio)
  
  
  def convert_and_save(real_audio_files, output_real, fake_audio_files, output_fake):
      for file in real_audio_files:
          wav_to_mel(file, output_real)
      print(str(len(real_audio_files)) + ' real files converted to spectrogram')
  
      for file in fake_audio_files:
          wav_to_mel(file, output_fake)
      print(str(len(fake_audio_files)) + ' fake files converted to spectrogram')
  
  
  def split_title_line(title_text, max_words=5):
      """
      A function that splits any string based on specific character
      (returning it with the string), with maximum number of words on it
      """
      seq = title_text.split()
      return '\n'.join([' '.join(seq[i:i + max_words]) for i in range(0, len(seq), max_words)])
  
  
  def plot_spectrogram(pred_spectrogram, path, title=None, split_title=False, target_spectrogram=None, max_len=None,
                       auto_aspect=False):
      if max_len is not None:
          target_spectrogram = target_spectrogram[:max_len]
          pred_spectrogram = pred_spectrogram[:max_len]
  
      if split_title:
          title = split_title_line(title)
  
      fig = plt.figure(figsize=(10, 8))
      # Set common labels
      fig.text(0.5, 0.18, title, horizontalalignment='center', fontsize=16)
  
      # target spectrogram subplot
      if target_spectrogram is not None:
          ax1 = fig.add_subplot(311)
          ax2 = fig.add_subplot(312)
  
          if auto_aspect:
              im = ax1.imshow(np.rot90(target_spectrogram), aspect='auto', interpolation='none')
          else:
              im = ax1.imshow(np.rot90(target_spectrogram), interpolation='none')
          ax1.set_title('Target Mel-Spectrogram')
          fig.colorbar(mappable=im, shrink=0.65, orientation='horizontal', ax=ax1)
          ax2.set_title('Predicted Mel-Spectrogram')
      else:
          ax2 = fig.add_subplot(211)
  
      if auto_aspect:
          im = ax2.imshow(np.rot90(pred_spectrogram), aspect='auto', interpolation='none')
      else:
          im = ax2.imshow(np.rot90(pred_spectrogram), interpolation='none')
      fig.colorbar(mappable=im, shrink=0.65, orientation='horizontal', ax=ax2)
  
      plt.tight_layout()
      plt.savefig(path, format='png')
      plt.close()
  
  
  def process_audio_files(filename, dirpath):
      audio_array, sample_rate = librosa.load(os.path.join(dirpath, 'flac', filename), sr=16000)
      trim_audio_array, index = librosa.effects.trim(audio_array)
      mel_spec_array = melspectrogram(trim_audio_array, hparams=hparams).T
      # mel_spec_array = librosa.feature.melspectrogram(y=trim_audio_array, sr=sample_rate, n_mels=model_params['num_freq_bin']).T
      label_name = filename.split('_')[-1].split('.')[0]
      if (label_name == 'bonafide') or ('target' in label_name):
          label = 1
      elif label_name == 'spoof':
          label = 0
      else:
          label = None
      if label is None:
          print(f"Removing {filename} since it does not have label")
          os.remove(os.path.join(dirpath, 'flac', filename))
      return (mel_spec_array, label)
  
  
  def convert_audio_to_processed_list(input_audio_array_list, filename, dirpath):
      label_name = filename.split('_')[-1].split('.')[0]
      out_list = []
      if (label_name == 'spoof'):
          audio_array_list = [input_audio_array_list[0]]
          choose_random_one_ind = np.random.choice(np.arange(1,len(input_audio_array_list)))
          audio_array_list.append(input_audio_array_list[choose_random_one_ind])
          label = 0
      elif (label_name == 'bonafide') or ('target' in label_name):
          audio_array_list = input_audio_array_list
          label = 1
      else:
          audio_array_list = [input_audio_array_list[0]]
          label = None
  
      for audio_array in audio_array_list:
          trim_audio_array, index = librosa.effects.trim(audio_array)
          mel_spec_array = melspectrogram(trim_audio_array, hparams=hparams).T
          # mel_spec_array = librosa.feature.melspectrogram(y=trim_audio_array, sr=sample_rate, n_mels=model_params['num_freq_bin']).T
          if label is None:
              print(f"Removing {filename} since it does not have label")
              os.remove(os.path.join(dirpath, 'flac', filename))
          out_list.append([mel_spec_array, label])
      return out_list
  
  
  
  def process_audio_files_with_aug(filename, dirpath):
      sr = 16000
      audio_array, sample_rate = librosa.load(os.path.join(dirpath, 'flac', filename), sr=sr)
      aug_crop = naa.CropAug(sampling_rate=sr)
      audio_array_crop = aug_crop.augment(audio_array)
      aug_loud = naa.LoudnessAug(loudness_factor=(2, 5))
      audio_array_loud = aug_loud.augment(audio_array)
      aug_noise = naa.NoiseAug(noise_factor=0.03)
      audio_array_noise = aug_noise.augment(audio_array)
  
      audio_array_list= [audio_array,audio_array_crop,audio_array_loud,
                         audio_array_noise ]
  
      out_list = convert_audio_to_processed_list(audio_array_list, filename, dirpath)
  
      return out_list
  
  
  
  def preprocess_and_save_audio_from_ray_parallel(dirpath, mode, recompute=False, dir_num=None, isaug=False):
      if isaug:
          preproc_filename = f'{mode}_preproc_aug.npy'
      else:
          preproc_filename = f'{mode}_preproc.npy'
  
      # if mode != 'train':
      #     preproc_filename = f'{mode}_preproc.npy'
  
      if dir_num is not None:
          base_path = base_data_path[dir_num]
      else:
          base_path = base_data_path[0]
      if not os.path.isfile(os.path.join(f'{base_path}/preprocessed_data', preproc_filename)) or recompute:
          filenames = os.listdir(os.path.join(dirpath, 'flac'))
          num_cores = multiprocessing.cpu_count()-1
          if isaug:
              precproc_list_saved = Parallel(n_jobs=num_cores)(
                  delayed(process_audio_files_with_aug)(filename, dirpath) for filename in tqdm(filenames))
              # Flatten the list
              print(f"******original len of preproc_list: {len(precproc_list_saved)}")
              precproc_list = []
              for i in range(len(precproc_list_saved)):
                  precproc_list.extend(precproc_list_saved[i])
              # precproc_list = [item for sublist in precproc_list for item in sublist]
              print(f"******flattened len of preproc_list: {len(precproc_list)}")
          else:
              precproc_list = Parallel(n_jobs=num_cores)(
                  delayed(process_audio_files)(filename, dirpath) for filename in tqdm(filenames))
  
          precproc_list = [x for x in precproc_list if x[1] is not None]
  
          if not os.path.isdir(f'{base_path}/preprocessed_data'):
              os.mkdir(f'{base_path}/preprocessed_data')
          np.save(os.path.join(f'{base_path}/preprocessed_data', preproc_filename), precproc_list)
      else:
          print("Preprocessing already done!")
  
  
  def process_audio_files_inference(filename, dirpath, mode):
     audio_array, sample_rate = librosa.load(os.path.join(dirpath, mode, filename), sr=16000)
     trim_audio_array, index = librosa.effects.trim(audio_array)
     mel_spec_array = melspectrogram(trim_audio_array, hparams=hparams).T
     if mode == 'unlabeled':
         return mel_spec_array
     elif mode == 'real':
         label = 1
     elif mode == 'fake':
         label = 0
     return mel_spec_array, label
  
  
  def preprocess_from_ray_parallel_inference(dirpath, mode, use_parallel=True):
     filenames = os.listdir(os.path.join(dirpath, mode))
     if use_parallel:
         num_cores = multiprocessing.cpu_count()
         preproc_list = Parallel(n_jobs=num_cores)(
             delayed(process_audio_files_inference)(filename, dirpath, mode) for filename in tqdm(filenames))
     else:
         preproc_list=[]
         for filename in tqdm(filenames):
             preproc_list.append(process_audio_files_inference(filename, dirpath, mode))
     return preproc_list
  
  
  def preprocess_and_save_audio_from_ray(dirpath, mode, recompute=False):
      filenames = os.listdir(os.path.join(dirpath, 'flac'))
      if not os.path.isfile(os.path.join(f'{base_data_path}/preprocessed_data', f'{mode}_preproc.npy')) or recompute:
          precproc_list = []
          for filename in tqdm(filenames):
              audio_array, sample_rate = librosa.load(os.path.join(dirpath, 'flac', filename), sr=16000)
              trim_audio_array, index = librosa.effects.trim(audio_array)
              mel_spec_array = melspectrogram(trim_audio_array, hparams=hparams).T
              # mel_spec_array = librosa.feature.melspectrogram(y=trim_audio_array, sr=sample_rate, n_mels=model_params['num_freq_bin']).T
              label_name = filename.split('_')[-1].split('.')[0]
              if label_name == 'bonafide':
                  label = 1
              elif label_name == 'spoof':
                  label = 0
              else:
                  label = None
              if label is not None:
                  precproc_list.append((mel_spec_array, label))
              if label is None:
                  print("Removing {filename} since it does not have label")
                  os.remove(os.path.join(dirpath, 'flac', filename))
          if not os.path.isdir(f'{base_data_path}/preprocessed_data'):
              os.mkdir(f'{base_data_path}/preprocessed_data')
          np.save(os.path.join(f'{base_data_path}/preprocessed_data', f'{mode}_preproc.npy'), precproc_list)
          # np.save(os.path.join(dirpath, 'preproc', 'preproc.npy'), precproc_list)
      else:
          print("Preprocessing already done!")
  
  
  def preprocess_and_save_audio(dirpath, recompute=False):
      filenames = os.listdir(os.path.join(dirpath, 'flac'))
      if not os.path.isfile(os.path.join(dirpath, 'preproc', 'preproc.npy')) or recompute:
          precproc_list = []
          for filename in tqdm(filenames):
              audio_array, sample_rate = librosa.load(os.path.join(dirpath, 'flac', filename), sr=16000)
              trim_audio_array, index = librosa.effects.trim(audio_array)
              mel_spec_array = librosa.feature.melspectrogram(y=trim_audio_array, sr=sample_rate,
                                                              n_mels=model_params['num_freq_bin']).T
              label_name = filename.split('_')[-1].split('.')[0]
              if label_name == 'bonafide':
                  label = 1
              elif label_name == 'spoof':
                  label = 0
              else:
                  label = None
              if label is not None:
                  precproc_list.append((mel_spec_array, label))
              if label is None:
                  print("Removing {filename} since it does not have label")
                  os.remove(os.path.join(dirpath, 'flac', filename))
          if not os.path.isdir(os.path.join(dirpath, 'preproc')):
              os.mkdir(os.path.join(dirpath, 'preproc'))
          np.save(os.path.join(dirpath, 'preproc', 'preproc.npy'), precproc_list)
      else:
          print("Preprocessing already done!")
  
  
  def describe_array(arr):
      print(f"Mean duration: {arr.mean()}\n Standard Deviation: {arr.std()}\nNumber of Clips: {len(arr)}")
      plt.hist(arr, bins=40)
      plt.show()
  
  
  def get_durations_from_dir(audio_dir, file_extension='.wav'):
      durations = list()
      for root, dirs, filenames in os.walk(audio_dir):
          for file_name in filenames:
              if file_extension in file_name:
                  file_path = os.path.join(root, file_name)
                  audio = AudioSegment.from_wav(file_path)
                  duration = audio.duration_seconds
                  durations.append(duration)
      return np.array(durations)
  
  
  def get_zero_pad(batch_input):
      # find max length
      max_length = np.max([len(x) for x in batch_input])
      for i, arr in enumerate(batch_input):
          curr_length = len(arr)
          pad_length = max_length - curr_length
          if len(arr.shape) > 1:
              arr = np.concatenate([arr, np.zeros((pad_length, arr.shape[-1]))])
          else:
              arr = np.concatenate([arr, np.zeros((pad_length))])
          batch_input[i] = arr
      return batch_input
  
  
  def truncate_array(batch_input):
      min_arr_len = np.min([len(x) for x in batch_input])
      for i, arr in enumerate(batch_input):
          batch_input[i] = arr[:min_arr_len]
      return batch_input
  
  def random_truncate_array(batch_input):
      min_arr_len = np.min([len(x) for x in batch_input])
      for i, arr in enumerate(batch_input):
          upper_limit_start_point = len(arr)-min_arr_len
          if upper_limit_start_point>0:
              start_point = np.random.randint(0,upper_limit_start_point)
          else:
              start_point = 0
          batch_input[i] = arr[start_point:(start_point+min_arr_len)]
      return batch_input
  
  
  class f1_score_callback(keras.callbacks.Callback):
      def __init__(self, x_val_inp, y_val_inp, model_save_filename=None, save_model=True):
          self.x_val = x_val_inp
          self.y_val = y_val_inp
          self.model_save_filename = model_save_filename
          self.save_model = save_model
          self._val_f1 = 0
  
      def on_train_begin(self, logs={}):
          self.f1_score_value = []
  
      def on_epoch_end(self, epoch, logs={}):
          y_val = self.y_val
          datagen_val = DataGenerator(self.x_val, mode='test')
          y_pred = self.model.predict_generator(datagen_val, use_multiprocessing=False, max_queue_size=50)
          y_pred_labels = np.zeros((len(y_pred)))
          y_pred_labels[y_pred.flatten() > 0.5] = 1
  
          self._val_f1 = f1_score(y_val, y_pred_labels.astype(int))
          print(f"val_f1: {self._val_f1:.4f}")
  
          self.f1_score_value.append(self._val_f1)
          if self.save_model:
              if self._val_f1 >= max(self.f1_score_value):
                  print("F1 score has improved. Saving model.")
                  self.model.save(self.model_save_filename)
  
          try:
              foundations.log_metric('epoch_val_f1_score',self._val_f1)
              foundations.log_metric('best_f1_score', max(self.f1_score_value))
          except Exception as e:
              print(e)
  
  
          return
  
  
  class DataGenerator(keras.utils.Sequence):
      def __init__(self, x_set, y_set=None, sample_weights=None, batch_size=model_params['batch_size'], shuffle=False, mode='train'):
          self.x, self.y = x_set, y_set
          self.batch_size = batch_size
          self.shuffle = shuffle
          self.mode = mode
          self.sample_weights = sample_weights
          if self.mode != 'train':
              self.shuffle = False
          self.n = 0
          self.max = self.__len__()
  
      def __len__(self):
          return int(np.ceil(len(self.x) / float(self.batch_size)))
  
      def __getitem__(self, idx):
          batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size]
          batch_x = get_zero_pad(batch_x)
          # batch_x = random_truncate_array(batch_x)
          batch_x = np.array(batch_x)
          batch_x = batch_x.reshape((len(batch_x), -1, hparams.num_mels))
          if self.mode != 'test':
              batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size]
          # read your data here using the batch lists, batch_x and batch_y
          if self.mode == 'train':
              return np.array(batch_x), np.array(batch_y)
          if self.mode == 'val':
              return np.array(batch_x), np.array(batch_y)
          if self.mode == 'test':
              return np.array(batch_x)
  
      def __next__(self):
          if self.n >= self.max:
              self.n = 0
          result = self.__getitem__(self.n)
          self.n += 1
          return result
  
  
  def customPooling(x):
      target = x[1]
      inputs = x[0]
      maskVal = 0
      #getting the mask by observing the model's inputs
      mask = K.equal(inputs, maskVal)
      mask = K.all(mask, axis=-1, keepdims=True)
  
      #inverting the mask for getting the valid steps for each sample
      mask = 1 - K.cast(mask, K.floatx())
  
      #summing the valid steps for each sample
      stepsPerSample = K.sum(mask, axis=1, keepdims=False)
  
      #applying the mask to the target (to make sure you are summing zeros below)
      target = target * mask
  
      #calculating the mean of the steps (using our sum of valid steps as averager)
      means = K.sum(target, axis=1, keepdims=False) / stepsPerSample
  
      return means
  
  
  
  def build_custom_convnet():
      K.clear_session()
      image_input = Input(shape=(None, model_params['num_freq_bin']), name='image_input')
  
      num_conv_blocks = model_params['num_conv_blocks']
      init_neurons = model_params['num_conv_filters']
      spatial_dropout_fraction = model_params['spatial_dropout_fraction']
      num_dense_layers = model_params['num_dense_layers']
      num_dense_neurons = model_params['num_dense_neurons']
      learning_rate = model_params['learning_rate']
      convnet = []
      convnet_5 = []
      convnet_7 = []
      for ly in range(0, num_conv_blocks):
          if ly == 0:
              convnet.append(Conv1D(init_neurons, 3, strides=1, activation='linear', padding='causal')(image_input))
              convnet_5.append(Conv1D(init_neurons, 5, strides=1, activation='linear', padding='causal')(image_input))
              convnet_7.append(Conv1D(init_neurons, 7, strides=1, activation='linear', padding='causal')(image_input))
          else:
              convnet.append(
                  Conv1D(init_neurons * (ly * 2), 3, strides=1, activation='linear', padding='causal')(convnet[ly - 1]))
              convnet_5.append(
                  Conv1D(init_neurons * (ly * 2), 5, strides=1, activation='linear', padding='causal')(convnet_5[ly - 1]))
              convnet_7.append(
                  Conv1D(init_neurons * (ly * 2), 7, strides=1, activation='linear', padding='causal')(convnet_7[ly - 1]))
  
          convnet[ly] = LeakyReLU()(convnet[ly])
          convnet_5[ly] = LeakyReLU()(convnet_5[ly])
          convnet_7[ly] = LeakyReLU()(convnet_7[ly])
          if model_params['residual_con'] > 0 and (ly - model_params['residual_con']) >= 0:
              res_conv = Conv1D(init_neurons * (ly * 2), 1, strides=1, activation='linear', padding='same')(
                  convnet[ly - model_params['residual_con']])
              convnet[ly] = Add(name=f'residual_con_3_{ly}')([convnet[ly], res_conv])
              res_conv_5 = Conv1D(init_neurons * (ly * 2), 1, strides=1, activation='linear', padding='same')(
                  convnet_5[ly - model_params['residual_con']])
              convnet_5[ly] = Add(name=f'residual_con_5_{ly}')([convnet_5[ly], res_conv_5])
              res_conv_7 = Conv1D(init_neurons * (ly * 2), 1, strides=1, activation='linear', padding='same')(
                  convnet_7[ly - model_params['residual_con']])
              convnet_7[ly] = Add(name=f'residual_con_7_{ly}')([convnet_7[ly], res_conv_7])
  
          if ly<(num_conv_blocks-1):
              convnet[ly] = SpatialDropout1D(spatial_dropout_fraction)(convnet[ly])
              convnet_5[ly] = SpatialDropout1D(spatial_dropout_fraction)(convnet_5[ly])
              convnet_7[ly] = SpatialDropout1D(spatial_dropout_fraction)(convnet_7[ly])
  
      dense = Lambda(lambda x: customPooling(x))([image_input,convnet[ly]])
      dense_5 = Lambda(lambda x: customPooling(x))([image_input,convnet_5[ly]])
      dense_7 = Lambda(lambda x: customPooling(x))([image_input,convnet_7[ly]])
  
      dense = Concatenate()([dense, dense_5, dense_7])
  
      for layers in range(num_dense_layers):
          dense = Dense(num_dense_neurons, activation='linear')(dense)
          dense = BatchNormalization()(dense)
          dense = LeakyReLU()(dense)
          dense = Dropout(model_params['dense_dropout'])(dense)
      output_layer = Dense(1)(dense)
      output_layer = Activation('sigmoid')(output_layer)
      model = Model(inputs=image_input, outputs=output_layer)
      opt = optimizers.Adam(lr=learning_rate)
      try:
          model = multi_gpu_model(model, gpus=4)
      except:
          pass
      model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])
      return model
  
  
  
  class Discriminator_Model():
      def __init__(self, load_pretrained=False, saved_model_name=None, real_test_mode=False):
  
          if not os.path.exists(model_params['model_save_dir']):
              os.makedirs(model_params['model_save_dir'])
  
          if not load_pretrained:
              self.model = build_custom_convnet()
              self.model.summary()
          else:
              self.model = load_model(os.path.join(f"audio_detector_api/model_predict/{model_params['model_save_dir']}/",saved_model_name), custom_objects={'customPooling': customPooling})
          param_items = model_params.items()
          self.model_name = f"saved_model_{'_'.join(str(v) for k,v in model_params.items())}.h5"
          self.real_test_model_name = f"real_test_saved_model_{'_'.join(str(v) for k,v in model_params.items())}.h5"
          self.model_save_filename = os.path.join(f"./{model_params['model_save_dir']}", self.model_name)
          self.real_test_model_save_filename = os.path.join(f"./{model_params['model_save_dir']}", self.real_test_model_name)
          if real_test_mode:
              if run_on_foundations:
                  self.real_test_data_dir = "../data/inference_data/"
              else:
                  self.real_test_data_dir = "../data/inference_data/"
  
              # preprocess the files
              self.real_test_processed_data_real = preprocess_from_ray_parallel_inference(self.real_test_data_dir, "real", use_parallel=False)
              self.real_test_processed_data_fake = preprocess_from_ray_parallel_inference(self.real_test_data_dir,
                                                                                          "fake",
                                                                                          use_parallel=False)
  
              self.real_test_processed_data = self.real_test_processed_data_real + self.real_test_processed_data_fake
              self.real_test_processed_data = sorted(self.real_test_processed_data, key=lambda x: len(x[0]))
              self.real_test_features = [x[0] for x in self.real_test_processed_data]
              self.real_test_labels = [x[1] for x in self.real_test_processed_data]
              print(f"Length of real_test_processed_data: {len(self.real_test_processed_data)}")
  
      def train(self, xtrain, ytrain, xval, yval):
          callbacks = []
          tb = TensorBoard(log_dir='tflogs', write_graph=True, write_grads=False)
          callbacks.append(tb)
  
          try:
              foundations.set_tensorboard_logdir('tflogs')
          except:
              print("foundations command not found")
  
          es = EarlyStopping(monitor='val_loss', mode='min', patience=5, min_delta=0.0001,
                             verbose=1)
          callbacks.append(tb)
          callbacks.append(es)
  
          rp = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=2,
                                 verbose=1)
          callbacks.append(rp)
  
          f1_callback = f1_score_callback(xval, yval, model_save_filename=self.model_save_filename)
          callbacks.append(f1_callback)
  
          class_weights = {1: 5, 0: 1}
  
          train_generator = DataGenerator(xtrain, ytrain)
          validation_generator = DataGenerator(xval, yval)
          self.model.fit_generator(train_generator,
                                   steps_per_epoch = len(train_generator),
                                   epochs = model_params['epochs'],
                                   validation_data=validation_generator,
                                   callbacks = callbacks,
                                   shuffle = False,
                                   use_multiprocessing = True,
                                    verbose = 1,
                                   class_weight =class_weights)
  
          # self.model = load_model(self.model_save_filename, custom_objects={'customPooling': customPooling})
  
          try:
              foundations.save_artifact(self.model_save_filename, key='trained_model.h5')
          except:
              print("foundations command not found")
  
  
      def inference_on_real_data(self, threshold=0.5):
          datagen_val = DataGenerator(self.real_test_features, mode='test', batch_size=1)
          y_pred = self.model.predict_generator(datagen_val, use_multiprocessing=False, max_queue_size=50)
          y_pred_labels = np.zeros((len(y_pred)))
          y_pred_labels[y_pred.flatten() > threshold] = 1
          acc_score = accuracy_score(self.real_test_labels, y_pred_labels)
          f1_score_val = f1_score(self.real_test_labels, y_pred_labels)
          return acc_score, f1_score_val
  
  
      def get_labels_from_prob(self, y, threshold=0.5):
          y_pred_labels = np.zeros((len(y)))
          y = np.array(y)
          if isinstance(threshold, list):
              y_pred_labels[y.flatten() > threshold[0]] = 1
          else:
              y_pred_labels[y.flatten() > threshold] = 1
          return y_pred_labels
  
      def get_f1score_for_optimization(self, threshold, y_true, y_pred, ismin=False):
          y_pred_labels = self.get_labels_from_prob(y_pred, threshold=threshold)
          if ismin:
              return - f1_score(y_true, y_pred_labels)
          else:
              return f1_score(y_true, y_pred_labels)
  
      def predict_labels(self, x, threshold=0.5, raw_prob=False, batch_size=model_params['batch_size']):
          test_generator = DataGenerator(x, mode='test', batch_size=batch_size)
          y_pred = self.model.predict_generator(test_generator, steps=len(test_generator), max_queue_size=10)
          print(y_pred)
          if raw_prob:
              return y_pred
          else:
              y_pred_labels = self.get_labels_from_prob(y_pred, threshold=threshold)
              return y_pred_labels
  
      def optimize_threshold(self, xtrain, ytrain, xval, yval):
          ytrain_pred = self.predict_labels(xtrain, raw_prob=True)
          yval_pred = self.predict_labels(xval, raw_prob=True)
          self.opt_threshold = 0.5
          ytrain_pred_labels = self.get_labels_from_prob(ytrain_pred, threshold=self.opt_threshold)
          yval_pred_labels = self.get_labels_from_prob(yval_pred, threshold=self.opt_threshold)
          train_f1_score = f1_score(ytrain_pred_labels, ytrain)
          val_f1_score = f1_score(yval_pred_labels, yval)
          print(f"train f1 score: {train_f1_score}, val f1 score: {val_f1_score}")
  
          f1_train_partial = partial(self.get_f1score_for_optimization, y_true=ytrain.copy(), y_pred=ytrain_pred.copy(), ismin=True)
          n_searches = 50
          dim_0 = Real(low=0.2, high=0.8, name='dim_0')
          dimensions = [dim_0]
          search_result = gp_minimize(func=f1_train_partial,
                                      dimensions=dimensions,
                                      acq_func='gp_hedge',  # Expected Improvement.
                                      n_calls=n_searches,
                                      # n_jobs=n_cpu,
                                      verbose=False)
  
          self.opt_threshold = search_result.x
          if isinstance(self.opt_threshold,list):
              self.opt_threshold = self.opt_threshold[0]
          self.optimum_threshold_filename = f"model_threshold_{'_'.join(str(v) for k, v in model_params.items())}.npy"
          np.save(os.path.join(f"{model_params['model_save_dir']}",self.optimum_threshold_filename), self.opt_threshold)
          train_f1_score = self.get_f1score_for_optimization(self.opt_threshold, y_true=ytrain, y_pred=ytrain_pred)
          val_f1_score = self.get_f1score_for_optimization(self.opt_threshold, y_true=yval, y_pred=yval_pred )
          print(f"optimized train f1 score: {train_f1_score}, optimized val f1 score: {val_f1_score}")
  
  
  
      def evaluate(self, xtrain, ytrain, xval, yval, num_examples=1):
          ytrain_pred = self.predict_labels(xtrain, raw_prob=True)
          yval_pred = self.predict_labels(xval, raw_prob=True)
          try:
              self.optimum_threshold_filename = f"model_threshold_{'_'.join(str(v) for k, v in model_params.items())}.npy"
              self.opt_threshold = np.load(os.path.join(f"{model_params['model_save_dir']}",self.optimum_threshold_filename)).item()
              print(f"loaded optimum threshold: {self.opt_threshold}")
          except:
              self.opt_threshold = 0.5
  
  
          ytrain_pred_labels = self.get_labels_from_prob(ytrain_pred, threshold=self.opt_threshold)
          yval_pred_labels = self.get_labels_from_prob(yval_pred, threshold=self.opt_threshold)
  
          train_accuracy = accuracy_score(ytrain, ytrain_pred_labels)
          val_accuracy = accuracy_score(yval, yval_pred_labels)
  
          train_f1_score = f1_score(ytrain, ytrain_pred_labels)
          val_f1_score = f1_score(yval, yval_pred_labels)
          print (f"train accuracy: {train_accuracy}, train_f1_score: {train_f1_score},"
                 f"val accuracy: {val_accuracy}, val_f1_score: {val_f1_score} ")
  
          try:
              foundations.log_metric('train_accuracy',np.round(train_accuracy,2))
              foundations.log_metric('val_accuracy', np.round(val_accuracy,2))
              foundations.log_metric('train_f1_score', np.round(train_f1_score,2))
              foundations.log_metric('val_f1_score', np.round(val_f1_score,2))
              foundations.log_metric('optimum_threshold', np.round(self.opt_threshold,2))
          except Exception as e:
              print(e)
  
          # True Positive Example
          ind_tp = np.argwhere(np.equal((yval_pred_labels + yval).astype(int), 2)).reshape(-1, )
  
          # True Negative Example
          ind_tn = np.argwhere(np.equal((yval_pred_labels + yval).astype(int), 0)).reshape(-1, )
  
          # False Positive Example
          ind_fp =np.argwhere( np.greater(yval_pred_labels, yval)).reshape(-1, )
  
          # False Negative Example
          ind_fn = np.argwhere(np.greater(yval, yval_pred_labels)).reshape(-1, )
  
  
          path_to_save_spetrograms = './spectrograms'
          if not os.path.isdir(path_to_save_spetrograms):
              os.makedirs(path_to_save_spetrograms)
          specs_saved = os.listdir(path_to_save_spetrograms)
          if len(specs_saved)>0:
              for file_ in specs_saved:
                  os.remove(os.path.join(path_to_save_spetrograms,file_))
  
          ind_random_tp = np.random.choice(ind_tp, num_examples).reshape(-1,)
          tp_x = [xtrain[i] for i in ind_random_tp]
  
          ind_random_tn = np.random.choice(ind_tn, num_examples).reshape(-1,)
          tn_x = [xtrain[i] for i in ind_random_tn]
  
          ind_random_fp = np.random.choice(ind_fp, num_examples).reshape(-1,)
          fp_x = [xtrain[i] for i in ind_random_fp]
  
          ind_random_fn = np.random.choice(ind_fn, num_examples).reshape(-1,)
          fn_x = [xtrain[i] for i in ind_random_fn]
  
          print("Plotting spectrograms to show what the hell the model has learned")
          for i in range(num_examples):
              plot_spectrogram(tp_x[i], path=os.path.join(path_to_save_spetrograms, f'true_positive_{i}.png'))
              plot_spectrogram(tn_x[i], path=os.path.join(path_to_save_spetrograms,f'true_negative_{i}.png'))
              plot_spectrogram(fp_x[i], path=os.path.join(path_to_save_spetrograms,f'false_positive_{i}.png'))
              plot_spectrogram(fn_x[i], path=os.path.join(path_to_save_spetrograms,f'fale_negative_{i}.png'))
  
          try:
              foundations.save_artifact(os.path.join(path_to_save_spetrograms, f'true_positive_{i}.png'), key='true_positive_example')
              foundations.save_artifact(os.path.join(path_to_save_spetrograms,f'true_negative_{i}.png'), key='true_negative_example')
              foundations.save_artifact(os.path.join(path_to_save_spetrograms,f'false_positive_{i}.png'), key='false_positive_example')
              foundations.save_artifact(os.path.join(path_to_save_spetrograms,f'fale_negative_{i}.png'), key='false_negative_example')
  
          except Exception as e:
              print(e)
  
      def save(self, path):
          self.model.save(path)
  
  