code: import os
  
  os.environ['CUDA_VISIBLE_DEVICES'] = '3'
  
  import pyroomacoustics as pra
  import numpy as np
  from pydub import AudioSegment
  from sklearn.utils import shuffle
  from glob import glob
  import random
  import json
  from malaya_speech.train.model.conformer.model import Model as ConformerModel
  from malaya_speech.train.model import hubert, ctc
  import malaya_speech.train as train
  import malaya_speech.config
  import malaya_speech.augmentation.waveform as augmentation
  import malaya_speech
  import tensorflow as tf
  import os
  import string
  
  
  sr = 16000
  maxlen = 18
  minlen_text = 1
  prob_aug = 0.95
  
  unique_vocab = [''] + list(string.ascii_lowercase + string.digits) + [' ']
  
  
  def augment_room(y, scale=1.0):
      corners = np.array(
          [[0, 0], [0, 5 * scale], [3 * scale, 5 * scale], [3 * scale, 0]]
      ).T
      room = pra.Room.from_corners(
          corners,
          fs=sr,
          materials=pra.Material(0.2, 0.15),
          ray_tracing=True,
          air_absorption=True,
      )
      room.extrude(3.5, materials=pra.Material(0.2, 0.15))
      room.set_ray_tracing(
          receiver_radius=0.5, n_rays=1000, energy_thres=1e-5
      )
      room.add_source([1.5 * scale, 4 * scale, 0.5], signal=y)
      R = np.array([[1.5 * scale], [0.5 * scale], [0.5]])
      room.add_microphone(R)
      room.simulate()
      return room.mic_array.signals[0]
  
  
  def random_amplitude_threshold(sample, low=1, high=2, threshold=0.4):
      y_aug = sample.copy()
      dyn_change = np.random.uniform(low=low, high=high)
      y_aug[np.abs(y_aug) >= threshold] = (
          y_aug[np.abs(y_aug) >= threshold] * dyn_change
      )
      return np.clip(y_aug, -1, 1)
  
  
  def add_uniform_noise(
      sample, power=0.01, return_noise=False, scale=False
  ):
      y_noise = sample.copy()
      noise_amp = power * np.random.uniform() * np.amax(y_noise)
      noise = noise_amp * np.random.normal(size=y_noise.shape[0])
      y_noise = y_noise + noise
      if scale:
          y_noise = y_noise / (np.max(np.abs(y_noise)) + 1e-9)
      if return_noise:
          if scale:
              noise = noise / (np.max(np.abs(y_noise)) + 1e-9)
          return y_noise, noise
      else:
          return y_noise
  
  
  def calc(signal, add_uniform=True):
      choice = random.randint(0, 10)
      print('choice', choice)
      if choice == 0:
          x = augmentation.sox_augment_high(
              signal,
              min_bass_gain=random.randint(25, 50),
              reverberance=random.randint(0, 80),
              hf_damping=10,
              room_scale=random.randint(0, 50),
              negate=1,
          )
      if choice == 1:
          x = augmentation.sox_augment_high(
              signal,
              min_bass_gain=random.randint(25, 70),
              reverberance=random.randint(0, 80),
              hf_damping=10,
              room_scale=random.randint(0, 50),
              negate=0,
          )
      if choice == 2:
          x = augmentation.sox_augment_low(
              signal,
              min_bass_gain=random.randint(5, 30),
              reverberance=random.randint(0, 80),
              hf_damping=10,
              room_scale=random.randint(0, 50),
              negate=random.randint(0, 1),
          )
      if choice == 3:
          x = augmentation.sox_augment_combine(
              signal,
              min_bass_gain_high=random.randint(25, 70),
              min_bass_gain_low=random.randint(5, 30),
              reverberance=random.randint(0, 80),
              hf_damping=10,
              room_scale=random.randint(0, 90),
          )
      if choice == 4:
          x = augmentation.sox_reverb(
              signal,
              reverberance=random.randint(10, 80),
              hf_damping=10,
              room_scale=random.randint(10, 90),
          )
      if choice == 5:
          x = random_amplitude_threshold(
              signal, threshold=random.uniform(0.35, 0.8)
          )
      if choice == 6:
          x = augmentation.lowpass_filter(
              signal, sr=sr, cutoff=random.randint(200, 551)
          )
      if choice == 7:
          x = augmentation.highpass_filter(
              signal, sr=sr, cutoff=random.randint(551, 1653)
          )
      if choice == 8:
          x = augmentation.bandpass_filter(
              signal,
              sr=sr,
              cutoff_low=random.randint(200, 551),
              cutoff_high=random.randint(551, 1653),
          )
      if choice == 9:
          x = augment_room(signal)
      if choice == 10:
          x = signal
  
      if choice not in [5] and random.gauss(0.5, 0.14) > 0.6:
          x = random_amplitude_threshold(
              x, low=1.0, high=2.0, threshold=random.uniform(0.6, 0.9)
          )
  
      if random.gauss(0.5, 0.14) > 0.6 and add_uniform:
          x = add_uniform_noise(x, power=random.uniform(0.005, 0.015))
  
      return x
  
  
  def mp3_to_wav(file, sr=sr):
      audio = AudioSegment.from_file(file)
      audio = audio.set_frame_rate(sr).set_channels(1)
      sample = np.array(audio.get_array_of_samples())
      return malaya_speech.astype.int_to_float(sample), sr
  
  
  def generate(file):
      with open(file) as fopen:
          dataset = json.load(fopen)
      audios, cleaned_texts = dataset['X'], dataset['Y']
      while True:
          audios, cleaned_texts = shuffle(audios, cleaned_texts)
          for i in range(len(audios)):
              try:
                  if audios[i].endswith('.mp3'):
                      # print('found mp3', audios[i])
                      wav_data, _ = mp3_to_wav(audios[i])
                  else:
                      wav_data, _ = malaya_speech.load(audios[i], sr=sr)
  
                  if len(cleaned_texts[i]) < minlen_text:
                      # print(f'skipped text too short {audios[i]}')
                      continue
  
                  if (len(wav_data) / sr) > maxlen:
                      continue
  
                  t = [unique_vocab.index(c) for c in cleaned_texts[i]]
  
                  yield {
                      'waveforms': wav_data,
                      'waveforms_length': [len(wav_data)],
                      'targets': t,
                      'targets_length': [len(t)],
                  }
              except Exception as e:
                  print(e)
  
  
  def get_dataset(
      file,
      batch_size=12,
      shuffle_size=20,
      thread_count=24,
      maxlen_feature=1800,
  ):
      def get():
          dataset = tf.data.Dataset.from_generator(
              generate,
              {
                  'waveforms': tf.float32,
                  'waveforms_length': tf.int32,
                  'targets': tf.int32,
                  'targets_length': tf.int32,
              },
              output_shapes={
                  'waveforms': tf.TensorShape([None]),
                  'waveforms_length': tf.TensorShape([None]),
                  'targets': tf.TensorShape([None]),
                  'targets_length': tf.TensorShape([None]),
              },
              args=(file,),
          )
          dataset = dataset.prefetch(tf.contrib.data.AUTOTUNE)
          dataset = dataset.padded_batch(
              batch_size,
              padded_shapes={
                  'waveforms': tf.TensorShape([None]),
                  'waveforms_length': tf.TensorShape([None]),
                  'targets': tf.TensorShape([None]),
                  'targets_length': tf.TensorShape([None]),
              },
              padding_values={
                  'waveforms': tf.constant(0, dtype=tf.float32),
                  'waveforms_length': tf.constant(0, dtype=tf.int32),
                  'targets': tf.constant(0, dtype=tf.int32),
                  'targets_length': tf.constant(0, dtype=tf.int32),
              },
          )
          return dataset
  
      return get
  
  
  class Encoder:
      def __init__(self, config):
          self.config = config
          self.encoder = ConformerModel(**self.config)
  
      def __call__(self, x, input_mask, training=True):
          return self.encoder(x, training=training)
  
  
  total_steps = 2000000
  
  
  def model_fn(features, labels, mode, params):
      config_conformer = malaya_speech.config.conformer_tiny_encoder_config
      config_conformer['subsampling']['type'] = 'none'
      config_conformer['dropout'] = 0.0
      encoder = Encoder(config_conformer)
      cfg = hubert.HuBERTConfig(
          extractor_mode='layer_norm',
          dropout=0.0,
          attention_dropout=0.0,
          encoder_layerdrop=0.0,
          dropout_input=0.0,
          dropout_features=0.0,
          final_dim=128,
      )
      model = hubert.Model(cfg, encoder, ['pad', 'eos', 'unk'] + [str(i) for i in range(100)])
      X = features['waveforms']
      X_len = features['waveforms_length'][:, 0]
      targets = features['targets']
      targets_int32 = tf.cast(targets, tf.int32)
      targets_length = features['targets_length'][:, 0]
      r = model(X, padding_mask=X_len, features_only=True, mask=False)
      logits = tf.layers.dense(r['x'], len(unique_vocab) + 1)
      seq_lens = tf.reduce_sum(
          tf.cast(tf.logical_not(r['padding_mask']), tf.int32), axis=1
      )
      mean_error, sum_error, sum_weight = ctc.loss.ctc_loss(
          logits, seq_lens, targets_int32, targets_length
      )
      loss = mean_error
      accuracy = ctc.metrics.ctc_sequence_accuracy(
          logits, seq_lens, targets_int32, targets_length,
      )
  
      tf.identity(loss, 'train_loss')
      tf.identity(accuracy, name='train_accuracy')
  
      tf.summary.scalar('train_accuracy', accuracy)
  
      variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
      init_checkpoint = 'hubert-conformer-tiny/model.ckpt-1000000'
  
      assignment_map, initialized_variable_names = train.get_assignment_map_from_checkpoint(
          variables, init_checkpoint
      )
  
      tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
  
      if mode == tf.estimator.ModeKeys.TRAIN:
          train_op = train.optimizer.adamw.create_optimizer(
              loss,
              init_lr=5e-5,
              num_train_steps=total_steps,
              num_warmup_steps=100000,
              end_learning_rate=0.0,
              weight_decay_rate=0.01,
              beta_1=0.9,
              beta_2=0.999,
              epsilon=1e-6,
              clip_norm=1.0,
          )
          estimator_spec = tf.estimator.EstimatorSpec(
              mode=mode, loss=loss, train_op=train_op
          )
  
      elif mode == tf.estimator.ModeKeys.EVAL:
  
          estimator_spec = tf.estimator.EstimatorSpec(
              mode=tf.estimator.ModeKeys.EVAL,
              loss=loss,
              eval_metric_ops={
                  'accuracy': ctc.metrics.ctc_sequence_accuracy_estimator(
                      logits, seq_lens, targets_int32, targets_length
                  )
              },
          )
  
      return estimator_spec
  
  
  train_hooks = [
      tf.train.LoggingTensorHook(
          ['train_accuracy', 'train_loss'], every_n_iter=1
      )
  ]
  train_dataset = get_dataset('bahasa-asr-train-combined.json')
  dev_dataset = get_dataset('bahasa-asr-test.json')
  
  train.run_training(
      train_fn=train_dataset,
      model_fn=model_fn,
      model_dir='hubert-conformer-tiny-ctc-char',
      num_gpus=1,
      log_step=1,
      save_checkpoint_step=20000,
      max_steps=total_steps,
      eval_fn=dev_dataset,
      train_hooks=train_hooks,
  )
   import os
  
  os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '../gcs/mesolitica-storage.json'
  os.environ['CUDA_VISIBLE_DEVICES'] = '1'
  
  import tensorflow as tf
  import malaya_speech.train as train
  import malaya_speech.train.model.vggvox_v2 as vggvox_v2
  import malaya_speech
  from glob import glob
  import librosa
  import numpy as np
  
  
  def lin_spectogram_from_wav(wav, hop_length, win_length, n_fft=1024):
      linear = librosa.stft(
          wav, n_fft=n_fft, win_length=win_length, hop_length=hop_length
      )  # linear spectrogram
      return linear.T
  
  
  def load_data(
      wav,
      win_length=400,
      sr=16000,
      hop_length=50,
      n_fft=512,
      spec_len=250,
      mode='train',
  ):
      linear_spect = lin_spectogram_from_wav(wav, hop_length, win_length, n_fft)
      mag, _ = librosa.magphase(linear_spect)  # magnitude
      mag_T = mag.T
      freq, time = mag_T.shape
      if mode == 'train':
          if time > spec_len:
              randtime = np.random.randint(0, time - spec_len)
              spec_mag = mag_T[:, randtime: randtime + spec_len]
          else:
              spec_mag = np.pad(mag_T, ((0, 0), (0, spec_len - time)), 'constant')
      else:
          spec_mag = mag_T
      # preprocessing, subtract mean, divided by time-wise var
      mu = np.mean(spec_mag, 0, keepdims=True)
      std = np.std(spec_mag, 0, keepdims=True)
      return (spec_mag - mu) / (std + 1e-5)
  
  
  DIMENSION = 257
  
  
  def calc(v):
  
      r = load_data(v, mode='eval')
      return r
  
  
  def preprocess_inputs(example):
      s = tf.compat.v1.numpy_function(calc, [example['inputs']], tf.float32)
  
      s = tf.reshape(s, (DIMENSION, -1, 1))
      example['inputs'] = s
  
      return example
  
  
  def parse(serialized_example):
  
      data_fields = {
          'inputs': tf.VarLenFeature(tf.float32),
          'targets': tf.VarLenFeature(tf.int64),
      }
      features = tf.parse_single_example(
          serialized_example, features=data_fields
      )
      for k in features.keys():
          features[k] = features[k].values
  
      features = preprocess_inputs(features)
  
      keys = list(features.keys())
      for k in keys:
          if k not in ['inputs', 'targets']:
              features.pop(k, None)
  
      return features
  
  
  def get_dataset(files, batch_size=32, shuffle_size=1024, thread_count=24):
      def get():
          dataset = tf.data.TFRecordDataset(files)
          dataset = dataset.map(parse, num_parallel_calls=thread_count)
          dataset = dataset.shuffle(shuffle_size)
          dataset = dataset.padded_batch(
              batch_size,
              padded_shapes={
                  'inputs': tf.TensorShape([DIMENSION, None, 1]),
                  'targets': tf.TensorShape([None]),
              },
              padding_values={
                  'inputs': tf.constant(0, dtype=tf.float32),
                  'targets': tf.constant(0, dtype=tf.int64),
              },
          )
          dataset = dataset.repeat()
          return dataset
  
      return get
  
  
  learning_rate = 1e-5
  init_checkpoint = '../vggvox-speaker-identification/v2/vggvox.ckpt'
  
  
  def model_fn(features, labels, mode, params):
      Y = tf.cast(features['targets'][:, 0], tf.int32)
      model = vggvox_v2.Model(features['inputs'], num_class=2, mode='train')
  
      logits = model.logits
      loss = tf.reduce_mean(
          tf.nn.sparse_softmax_cross_entropy_with_logits(
              logits=logits, labels=Y
          )
      )
  
      tf.identity(loss, 'train_loss')
  
      accuracy = tf.metrics.accuracy(
          labels=Y, predictions=tf.argmax(logits, axis=1)
      )
  
      tf.identity(accuracy[1], name='train_accuracy')
      tf.summary.scalar('train_accuracy', accuracy[1])
  
      variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
      variables = [v for v in variables if 'prediction' not in v.name]
  
      assignment_map, initialized_variable_names = train.get_assignment_map_from_checkpoint(
          variables, init_checkpoint
      )
  
      tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
  
      if mode == tf.estimator.ModeKeys.TRAIN:
          global_step = tf.train.get_or_create_global_step()
          optimizer = tf.train.AdamOptimizer(learning_rate)
          train_op = optimizer.minimize(loss, global_step=global_step)
          estimator_spec = tf.estimator.EstimatorSpec(
              mode=mode, loss=loss, train_op=train_op
          )
  
      elif mode == tf.estimator.ModeKeys.EVAL:
  
          estimator_spec = tf.estimator.EstimatorSpec(
              mode=tf.estimator.ModeKeys.EVAL,
              loss=loss,
              eval_metric_ops={'accuracy': accuracy},
          )
  
      return estimator_spec
  
  
  train_hooks = [
      tf.train.LoggingTensorHook(
          ['train_accuracy', 'train_loss'], every_n_iter=1
      )
  ]
  
  files = tf.io.gfile.glob(
      'gs://mesolitica-general/speaker-change/data/*.tfrecords'
  )
  train_dataset = get_dataset(files)
  
  save_directory = 'output-vggvox-v2-speaker-change'
  
  train.run_training(
      train_fn=train_dataset,
      model_fn=model_fn,
      model_dir=save_directory,
      num_gpus=1,
      log_step=1,
      save_checkpoint_step=25000,
      max_steps=300000,
      train_hooks=train_hooks,
  )
   import os
  
  os.environ['CUDA_VISIBLE_DEVICES'] = '2,3'
  
  import tensorflow as tf
  import malaya_speech
  import malaya_speech.augmentation.waveform as augmentation
  import malaya_speech.augmentation.spectrogram as mask_augmentation
  import malaya_speech.train.model.medium_jasper as jasper
  import malaya_speech.train.model.ctc as ctc
  import malaya_speech.train as train
  from malaya_speech.train.model.quartznet import layer, abstract
  import numpy as np
  import random
  from glob import glob
  import json
  
  with open('malaya-speech-sst-vocab.json') as fopen:
      unique_vocab = json.load(fopen) + ['{', '}', '[']
  
  parameters = {
      'optimizer_params': {},
      'lr_policy_params': {
          'learning_rate': 1e-4,
          'min_lr': 1e-6,
          'warmup_steps': 0,
          'decay_steps': 500_000,
      },
  }
  
  
  def learning_rate_scheduler(global_step):
      return train.schedule.cosine_decay(
          global_step, **parameters['lr_policy_params']
      )
  
  
  featurizer = malaya_speech.tf_featurization.STTFeaturizer(
      normalize_per_feature=True
  )
  n_mels = featurizer.num_feature_bins
  
  
  def mel_augmentation(features):
  
      features = mask_augmentation.mask_frequency(features, width_freq_mask=15)
      features = mask_augmentation.mask_time(
          features, width_time_mask=int(features.shape[0] * 0.05)
      )
      return features
  
  
  def preprocess_inputs(example):
      s = featurizer.vectorize(example['waveforms'])
      s = tf.reshape(s, (-1, n_mels))
      s = tf.compat.v1.numpy_function(mel_augmentation, [s], tf.float32)
      mel_fbanks = tf.reshape(s, (-1, n_mels))
      length = tf.cast(tf.shape(mel_fbanks)[0], tf.int32)
      length = tf.expand_dims(length, 0)
      example['inputs'] = mel_fbanks
      example['inputs_length'] = length
  
      return example
  
  
  def parse(serialized_example):
  
      data_fields = {
          'waveforms': tf.VarLenFeature(tf.float32),
          'targets': tf.VarLenFeature(tf.int64),
      }
      features = tf.parse_single_example(
          serialized_example, features=data_fields
      )
      for k in features.keys():
          features[k] = features[k].values
  
      features = preprocess_inputs(features)
  
      keys = list(features.keys())
      for k in keys:
          if k not in ['waveforms', 'inputs', 'inputs_length', 'targets']:
              features.pop(k, None)
  
      return features
  
  
  def get_dataset(
      path,
      batch_size=32,
      shuffle_size=32,
      thread_count=24,
      maxlen_feature=1800,
  ):
      def get():
          files = glob(path)
          dataset = tf.data.TFRecordDataset(files)
          dataset = dataset.shuffle(shuffle_size)
          dataset = dataset.repeat()
          dataset = dataset.prefetch(tf.contrib.data.AUTOTUNE)
          dataset = dataset.map(parse, num_parallel_calls=thread_count)
          dataset = dataset.padded_batch(
              batch_size,
              padded_shapes={
                  'waveforms': tf.TensorShape([None]),
                  'inputs': tf.TensorShape([None, n_mels]),
                  'inputs_length': tf.TensorShape([None]),
                  'targets': tf.TensorShape([None]),
              },
              padding_values={
                  'waveforms': tf.constant(0, dtype=tf.float32),
                  'inputs': tf.constant(0, dtype=tf.float32),
                  'inputs_length': tf.constant(0, dtype=tf.int32),
                  'targets': tf.constant(0, dtype=tf.int64),
              },
          )
          return dataset
  
      return get
  
  
  def model_fn(features, labels, mode, params):
  
      model = jasper.Model(
          features['inputs'], features['inputs_length'][:, 0], training=True
      )
      logits = tf.layers.dense(model.logits['outputs'], len(unique_vocab) + 1)
      seq_lens = model.logits['src_length']
  
      targets_int32 = tf.cast(features['targets'], tf.int32)
  
      mean_error, sum_error, sum_weight = ctc.loss.ctc_loss(
          logits, targets_int32, seq_lens
      )
  
      loss = mean_error
      accuracy = ctc.metrics.ctc_sequence_accuracy(
          logits, targets_int32, seq_lens
      )
  
      tf.identity(loss, 'train_loss')
      tf.identity(accuracy, name='train_accuracy')
  
      tf.summary.scalar('train_accuracy', accuracy)
  
      if mode == tf.estimator.ModeKeys.TRAIN:
          train_op = train.optimizer.optimize_loss(
              loss,
              tf.train.AdamOptimizer,
              parameters['optimizer_params'],
              learning_rate_scheduler,
              summaries=['learning_rate', 'loss_scale'],
              larc_params=parameters.get('larc_params', None),
              loss_scaling=parameters.get('loss_scaling', 1.0),
              loss_scaling_params=parameters.get('loss_scaling_params', None),
          )
          estimator_spec = tf.estimator.EstimatorSpec(
              mode=mode, loss=loss, train_op=train_op
          )
  
      elif mode == tf.estimator.ModeKeys.EVAL:
  
          estimator_spec = tf.estimator.EstimatorSpec(
              mode=tf.estimator.ModeKeys.EVAL,
              loss=loss,
              eval_metric_ops={
                  'accuracy': ctc.metrics.ctc_sequence_accuracy_estimator(
                      logits, targets_int32, seq_lens
                  )
              },
          )
  
      return estimator_spec
  
  
  train_hooks = [
      tf.train.LoggingTensorHook(
          ['train_accuracy', 'train_loss'], every_n_iter=1
      )
  ]
  train_dataset = get_dataset(
      '../speech-bahasa/bahasa-asr/data/bahasa-asr-train-*'
  )
  dev_dataset = get_dataset(
      '../speech-bahasa/bahasa-asr-test/data/bahasa-asr-dev-*'
  )
  
  train.run_training(
      train_fn=train_dataset,
      model_fn=model_fn,
      model_dir='asr-medium-jasper-ctc',
      num_gpus=2,
      log_step=1,
      save_checkpoint_step=5000,
      max_steps=parameters['lr_policy_params']['decay_steps'],
      eval_fn=dev_dataset,
      train_hooks=train_hooks,
  )
  
apis: numpy.random.uniform tensorflow.summary.scalar tensorflow.train.LoggingTensorHook tensorflow.logical_not tensorflow.identity tensorflow.get_collection numpy.random.normal numpy.abs sklearn.utils.shuffle tensorflow.estimator.EstimatorSpec tensorflow.cast numpy.amax numpy.clip tensorflow.TensorShape tensorflow.train.init_from_checkpoint numpy.array tensorflow.constant
apis: tensorflow.summary.scalar tensorflow.data.TFRecordDataset tensorflow.VarLenFeature tensorflow.reshape tensorflow.nn.sparse_softmax_cross_entropy_with_logits tensorflow.identity tensorflow.train.LoggingTensorHook tensorflow.train.init_from_checkpoint tensorflow.train.get_or_create_global_step tensorflow.constant numpy.mean tensorflow.parse_single_example tensorflow.get_collection tensorflow.estimator.EstimatorSpec tensorflow.cast tensorflow.TensorShape numpy.std tensorflow.io.gfile.glob numpy.pad tensorflow.train.AdamOptimizer tensorflow.compat.v1.numpy_function tensorflow.argmax numpy.random.randint
apis: tensorflow.summary.scalar tensorflow.train.LoggingTensorHook tensorflow.VarLenFeature tensorflow.data.TFRecordDataset tensorflow.shape tensorflow.reshape tensorflow.parse_single_example tensorflow.expand_dims tensorflow.estimator.EstimatorSpec tensorflow.cast tensorflow.TensorShape tensorflow.compat.v1.numpy_function tensorflow.identity tensorflow.constant