code: def load_data(
      data_filename,
      disable_caching=False,
      estimate_audio=False,
      vocab=None,
      audio_base_path=None,
      comparison_mode=False,
      names=None,
  ):
      if comparison_mode:
          if names is None:
              logging.error(f"Please, specify names of compared models")
          name_1, name_2 = names
  
      if not comparison_mode:
          if vocab is not None:
              # load external vocab
              vocabulary_ext = {}
              with open(vocab, "r") as f:
                  for line in f:
                      if "\t" in line:
                          # parse word from TSV file
                          word = line.split("\t")[0]
                      else:
                          # assume each line contains just a single word
                          word = line.strip()
                      vocabulary_ext[word] = 1
  
          if not disable_caching:
              pickle_filename = data_filename.split(".json")[0]
              json_mtime = datetime.datetime.fromtimestamp(
                  os.path.getmtime(data_filename)
              )
              timestamp = json_mtime.strftime("%Y%m%d_%H%M")
              pickle_filename += "_" + timestamp + ".pkl"
              if os.path.exists(pickle_filename):
                  with open(pickle_filename, "rb") as f:
                      (
                          data,
                          wer,
                          cer,
                          wmr,
                          mwa,
                          num_hours,
                          vocabulary_data,
                          alphabet,
                          metrics_available,
                      ) = pickle.load(f)
                  if vocab is not None:
                      for item in vocabulary_data:
                          item["OOV"] = (
                              item["word"] not in vocabulary_ext
                          )
                  if estimate_audio:
                      for item in data:
                          filepath = absolute_audio_filepath(
                              item["audio_filepath"], audio_base_path
                          )
                          signal, sr = librosa.load(
                              path=filepath, sr=None
                          )
                          bw = eval_bandwidth(signal, sr)
                          item["freq_bandwidth"] = int(bw)
                          item["level_db"] = 20 * np.log10(
                              np.max(np.abs(signal))
                          )
                  with open(pickle_filename, "wb") as f:
                      pickle.dump(
                          [
                              data,
                              wer,
                              cer,
                              wmr,
                              mwa,
                              num_hours,
                              vocabulary_data,
                              alphabet,
                              metrics_available,
                          ],
                          f,
                          pickle.HIGHEST_PROTOCOL,
                      )
                  return (
                      data,
                      wer,
                      cer,
                      wmr,
                      mwa,
                      num_hours,
                      vocabulary_data,
                      alphabet,
                      metrics_available,
                  )
  
      data = []
      wer_count = 0
      cer_count = 0
      wmr_count = 0
      wer = 0
      cer = 0
      wmr = 0
      mwa = 0
      num_hours = 0
      match_vocab_1 = defaultdict(lambda: 0)
      match_vocab_2 = defaultdict(lambda: 0)
  
      def append_data(
          data_filename,
          estimate_audio,
          field_name="pred_text",
      ):
          data = []
          wer_dist = 0.0
          wer_count = 0
          cer_dist = 0.0
          cer_count = 0
          wmr_count = 0
          wer = 0
          cer = 0
          wmr = 0
          mwa = 0
          num_hours = 0
          vocabulary = defaultdict(lambda: 0)
          alphabet = set()
          match_vocab = defaultdict(lambda: 0)
  
          sm = difflib.SequenceMatcher()
          metrics_available = False
          with open(data_filename, "r", encoding="utf8") as f:
              for line in tqdm.tqdm(f):
                  item = json.loads(line)
                  if not isinstance(item["text"], str):
                      item["text"] = ""
                  num_chars = len(item["text"])
                  orig = item["text"].split()
                  num_words = len(orig)
                  for word in orig:
                      vocabulary[word] += 1
                  for char in item["text"]:
                      alphabet.add(char)
                  num_hours += item["duration"]
  
                  if field_name in item:
                      metrics_available = True
                      pred = item[field_name].split()
                      measures = jiwer.compute_measures(
                          item["text"], item[field_name]
                      )
                      word_dist = (
                          measures["substitutions"]
                          + measures["insertions"]
                          + measures["deletions"]
                      )
                      char_dist = editdistance.eval(
                          item["text"], item[field_name]
                      )
                      wer_dist += word_dist
                      cer_dist += char_dist
                      wer_count += num_words
                      cer_count += num_chars
  
                      sm.set_seqs(orig, pred)
                      for m in sm.get_matching_blocks():
                          for word_idx in range(m[0], m[0] + m[2]):
                              match_vocab[orig[word_idx]] += 1
                      wmr_count += measures["hits"]
                  else:
                      if comparison_mode:
                          if field_name != "pred_text":
                              if field_name == name_1:
                                  logging.error(
                                      f"The .json file has no field with name: {name_1}"
                                  )
                                  exit()
                              if field_name == name_2:
                                  logging.error(
                                      f"The .json file has no field with name: {name_2}"
                                  )
                                  exit()
                  data.append(
                      {
                          "audio_filepath": item["audio_filepath"],
                          "duration": round(item["duration"], 2),
                          "num_words": num_words,
                          "num_chars": num_chars,
                          "word_rate": round(
                              num_words / item["duration"], 2
                          ),
                          "char_rate": round(
                              num_chars / item["duration"], 2
                          ),
                          "text": item["text"],
                      }
                  )
                  if metrics_available:
                      data[-1][field_name] = item[field_name]
                      if num_words == 0:
                          num_words = 1e-9
                      if num_chars == 0:
                          num_chars = 1e-9
                      data[-1]["WER"] = round(
                          word_dist / num_words * 100.0, 2
                      )
                      data[-1]["CER"] = round(
                          char_dist / num_chars * 100.0, 2
                      )
                      data[-1]["WMR"] = round(
                          measures["hits"] / num_words * 100.0, 2
                      )
                      data[-1]["I"] = measures["insertions"]
                      data[-1]["D"] = measures["deletions"]
                      data[-1]["D-I"] = (
                          measures["deletions"] - measures["insertions"]
                      )
                  if estimate_audio:
                      filepath = absolute_audio_filepath(
                          item["audio_filepath"], data_filename
                      )
                      signal, sr = librosa.load(path=filepath, sr=None)
                      bw = eval_bandwidth(signal, sr)
                      item["freq_bandwidth"] = int(bw)
                      item["level_db"] = 20 * np.log10(
                          np.max(np.abs(signal))
                      )
                  for k in item:
                      if k not in data[-1]:
                          data[-1][k] = item[k]
  
              vocabulary_data = [
                  {"word": word, "count": vocabulary[word]}
                  for word in vocabulary
              ]
              return (
                  vocabulary_data,
                  metrics_available,
                  data,
                  wer_dist,
                  wer_count,
                  cer_dist,
                  cer_count,
                  wmr_count,
                  wer,
                  cer,
                  wmr,
                  mwa,
                  num_hours,
                  vocabulary,
                  alphabet,
                  match_vocab,
              )
  
      (
          vocabulary_data,
          metrics_available,
          data,
          wer_dist,
          wer_count,
          cer_dist,
          cer_count,
          wmr_count,
          wer,
          cer,
          wmr,
          mwa,
          num_hours,
          vocabulary,
          alphabet,
          match_vocab,
      ) = append_data(data_filename, estimate_audio, field_name=fld_nm)
      if comparison_mode:
          (
              vocabulary_data_1,
              metrics_available_1,
              data_1,
              wer_dist_1,
              wer_count_1,
              cer_dist_1,
              cer_count_1,
              wmr_count_1,
              wer_1,
              cer_1,
              wmr_1,
              mwa_1,
              num_hours_1,
              vocabulary_1,
              alphabet_1,
              match_vocab_1,
          ) = append_data(
              data_filename, estimate_audio, field_name=name_1
          )
          (
              vocabulary_data_2,
              metrics_available_2,
              data_2,
              wer_dist_2,
              wer_count_2,
              cer_dist_2,
              cer_count_2,
              wmr_count_2,
              wer_2,
              cer_2,
              wmr_2,
              mwa_2,
              num_hours_2,
              vocabulary_2,
              alphabet_2,
              match_vocab_2,
          ) = append_data(
              data_filename, estimate_audio, field_name=name_2
          )
  
      if not comparison_mode:
          if vocab is not None:
              for item in vocabulary_data:
                  item["OOV"] = item["word"] not in vocabulary_ext
  
      if metrics_available or comparison_mode:
          if metrics_available:
              wer = wer_dist / wer_count * 100.0
              cer = cer_dist / cer_count * 100.0
              wmr = wmr_count / wer_count * 100.0
          if comparison_mode:
              if metrics_available_1 and metrics_available_2:
                  wer_1 = wer_dist_1 / wer_count_1 * 100.0
                  cer_1 = cer_dist_1 / cer_count_1 * 100.0
                  wmr_1 = wmr_count_1 / wer_count_1 * 100.0
  
                  wer = wer_dist_2 / wer_count_2 * 100.0
                  cer = cer_dist_2 / cer_count_2 * 100.0
                  wmr = wmr_count_2 / wer_count_2 * 100.0
  
                  acc_sum_1 = 0
                  acc_sum_2 = 0
  
                  for item in vocabulary_data_1:
                      w = item["word"]
                      word_accuracy_1 = (
                          match_vocab_1[w] / vocabulary_1[w] * 100.0
                      )
                      acc_sum_1 += word_accuracy_1
                      item["accuracy_1"] = round(word_accuracy_1, 1)
                  mwa_1 = acc_sum_1 / len(vocabulary_data_1)
  
                  for item in vocabulary_data_2:
                      w = item["word"]
                      word_accuracy_2 = (
                          match_vocab_2[w] / vocabulary_2[w] * 100.0
                      )
                      acc_sum_2 += word_accuracy_2
                      item["accuracy_2"] = round(word_accuracy_2, 1)
                  mwa_2 = acc_sum_2 / len(vocabulary_data_2)
  
          acc_sum = 0
          for item in vocabulary_data:
              w = item["word"]
              word_accuracy = match_vocab[w] / vocabulary[w] * 100.0
              acc_sum += word_accuracy
              item["accuracy"] = round(word_accuracy, 1)
          mwa = acc_sum / len(vocabulary_data)
  
      num_hours /= 3600.0
  
      if not comparison_mode:
          if not disable_caching:
              with open(pickle_filename, "wb") as f:
                  pickle.dump(
                      [
                          data,
                          wer,
                          cer,
                          wmr,
                          mwa,
                          num_hours,
                          vocabulary_data,
                          alphabet,
                          metrics_available,
                      ],
                      f,
                      pickle.HIGHEST_PROTOCOL,
                  )
      if comparison_mode:
          return (
              data,
              wer,
              cer,
              wmr,
              mwa,
              num_hours,
              vocabulary_data,
              alphabet,
              metrics_available,
              data_1,
              wer_1,
              cer_1,
              wmr_1,
              mwa_1,
              num_hours_1,
              vocabulary_data_1,
              alphabet_1,
              metrics_available_1,
              data_2,
              wer_2,
              cer_2,
              wmr_2,
              mwa_2,
              num_hours_2,
              vocabulary_data_2,
              alphabet_2,
              metrics_available_2,
          )
  
      return (
          data,
          wer,
          cer,
          wmr,
          mwa,
          num_hours,
          vocabulary_data,
          alphabet,
          metrics_available,
      )
  
imports: import argparse
  import base64
  import csv
  import datetime
  import difflib
  import io
  import json
  import logging
  import math
  import operator
  import os
  import pickle
  from collections import defaultdict
  from os.path import expanduser
  from pathlib import Path
  import dash
  import dash_bootstrap_components
  import diff_match_patch
  import editdistance
  import jiwer
  import librosa
  import numpy
  import pandas
  import soundfile
  import tqdm
  from dash import dash_table, dcc, html
  from dash.dependencies import Input, Output, State
  from dash.exceptions import PreventUpdate
  from plotly import express
  from plotly import graph_objects
  from plotly.subplots import make_subplots
label_desc: Use image node_id 4 for calling a global function with example usage: load_data(data_filename, disable_caching, estimate_audio, vocab, audio_base_path, comparison_mode, names) and returns: data,  wer,  cer,  wmr,  mwa,  num_hours,  vocabulary_data,  alphabet,  metrics_available, data,  wer,  cer,  wmr,  mwa,  num_hours,  vocabulary_data,  alphabet,  metrics_available,  data_1,  wer_1,  cer_1,  wmr_1,  mwa_1,  num_hours_1,  vocabulary_data_1,  alphabet_1,  metrics_available_1,  data_2,  wer_2,  cer_2,  wmr_2,  mwa_2,  num_hours_2,  vocabulary_data_2,  alphabet_2,  metrics_available_2, vocabulary_data,  metrics_available,  data,  wer_dist,  wer_count,  cer_dist,  cer_count,  wmr_count,  wer,  cer,  wmr,  mwa,  num_hours,  vocabulary,  alphabet,  match_vocab, data,  wer,  cer,  wmr,  mwa,  num_hours,  vocabulary_data,  alphabet,  metrics_available