code: def run_inference(args):
      device = torch.device("cuda", 0)
      model = torch.jit.load(args.model)
      model.to(device)
      model.eval()
  
      bpe_model = spm.SentencePieceProcessor()
      bpe_model.load(args.bpe_model)
      vocabs = [
          bpe_model.id_to_piece(id)
          for id in range(bpe_model.get_piece_size())
      ]
      if args.using_cpu_decoder:
          cpu_decoder = ctc_decoder(
              lexicon=None,
              tokens=vocabs,
              lm=None,
              nbest=args.nbest,
              beam_size=args.beam_size,
              beam_size_token=args.beam_size_token,
              beam_threshold=args.beam_threshold,
              blank_token="<blk>",
              sil_token="<blk>",
          )
      else:
          assert (
              vocabs[0] == "<blk>"
          ), "idx of blank token has to be zero"
  
          cuda_decoder = cuda_ctc_decoder(
              vocabs,
              nbest=args.nbest,
              beam_size=args.beam_size,
              blank_skip_threshold=args.blank_skip_threshold,
          )
  
      dataset = torchaudio.datasets.LIBRISPEECH(
          args.librispeech_path, url=args.split, download=True
      )
  
      total_edit_distance, oracle_edit_distance, total_length = 0, 0, 0
  
      data_loader = DataLoader(
          dataset,
          batch_size=args.batch_size,
          num_workers=4,
          pin_memory=True,
          collate_fn=collate_wrapper,
      )
  
      decoding_duration = 0
      for idx, batch in enumerate(data_loader):
          waveforms, transcripts = batch
          waveforms = [wave.to(device) for wave in waveforms]
          features = [
              torchaudio.compliance.kaldi.fbank(
                  wave, num_mel_bins=80, snip_edges=False
              )
              for wave in waveforms
          ]
          feature_lengths = [f.size(0) for f in features]
  
          features = pad_sequence(
              features,
              batch_first=True,
              padding_value=torch.log(torch.tensor(1e-10)),
          )
          feature_lengths = torch.tensor(feature_lengths, device=device)
  
          encoder_out, encoder_out_lens = model.encoder(
              x=features,
              x_lens=feature_lengths,
          )
          nnet_output = model.ctc_output(encoder_out)
          log_prob = torch.nn.functional.log_softmax(nnet_output, -1)
  
          decoding_start = time.perf_counter()
          preds = []
          if args.using_cpu_decoder:
              results = cpu_decoder(log_prob.cpu())
              duration = time.perf_counter() - decoding_start
              for i in range(len(results)):
                  ith_preds = bpe_model.decode(
                      [
                          results[i][j].tokens.tolist()
                          for j in range(len(results[i]))
                      ]
                  )
                  ith_preds = [
                      pred.lower().split() for pred in ith_preds
                  ]
                  preds.append(ith_preds)
          else:
              results = cuda_decoder(
                  log_prob, encoder_out_lens.to(torch.int32)
              )
              duration = time.perf_counter() - decoding_start
              for i in range(len(results)):
                  ith_preds = bpe_model.decode(
                      [
                          results[i][j].tokens
                          for j in range(len(results[i]))
                      ]
                  )
                  ith_preds = [
                      pred.lower().split() for pred in ith_preds
                  ]
                  preds.append(ith_preds)
          decoding_duration += duration
  
          for transcript, nbest_pred in zip(transcripts, preds):
              total_edit_distance += (
                  torchaudio.functional.edit_distance(
                      transcript.split(), nbest_pred[0]
                  )
              )
              oracle_edit_distance += min(
                  [
                      torchaudio.functional.edit_distance(
                          transcript.split(), nbest_pred[i]
                      )
                      for i in range(len(nbest_pred))
                  ]
              )
              total_length += len(transcript.split())
  
          if idx % 10 == 0:
              logger.info(
                  f"Processed elem {idx}; "
                  f"WER: {total_edit_distance / total_length}, "
                  f"Oracle WER: {oracle_edit_distance / total_length}, ",
                  f"decoding time for batch size {args.batch_size}: {duration}",
              )
  
      logger.info(
          f"Final WER: {total_edit_distance / total_length}, ",
          f"Oracle WER: {oracle_edit_distance / total_length}, ",
          f"time for decoding {decoding_duration} [sec].",
      )
  
file_path: audio/examples/asr/librispeech_cuda_ctc_decoder/inference.py
imports: import argparse
  import logging
  import time
  import sentencepiece
  import torch
  import torchaudio
  from torch.nn.utils.rnn import pad_sequence
  from torch.utils.data import DataLoader
  from torchaudio.models.decoder import ctc_decoder, cuda_ctc_decoder
label_desc: Use image node_id 2 for calling a global function with example usage: run_inference(args) without return types