code: def read_train_file(
      path,
      is_aggregate_tokenizer: bool = False,
      verbose: int = 0,
  ):
      lines_read = 0
      text_dataset, lang_dataset = [], []
      if path[-8:] == ".json.gz":  # for Common Crawl dataset
          fin = gzip.open(path, "r")
      else:
          fin = open(path, "r", encoding="utf-8")
  
      if verbose > 0:
          reader = tqdm(
              iter(lambda: fin.readline(), ""),
              desc="Read 0 lines",
              unit=" lines",
          )
      else:
          reader = fin
  
      for line in reader:
          lang = None
          if line:
              if path[-8:] == ".json.gz":  # for Common Crawl dataset
                  line = json.loads(line.decode("utf-8"))["text"]
              elif path.endswith(".json"):
                  jline = json.loads(line)
                  line = jline["text"]
                  if is_aggregate_tokenizer:
                      lang = jline["lang"]
  
              line_list = line.split("\n")
  
              line = " ".join(line_list)
              if line:
                  text_dataset.append(line)
                  if lang:
                      lang_dataset.append(lang)
                  lines_read += 1
                  if verbose > 0 and lines_read % 100000 == 0:
                      reader.set_description(f"Read {lines_read} lines")
          else:
              break
      fin.close()
      if is_aggregate_tokenizer:
          assert len(text_dataset) == len(
              lang_dataset
          ), f"text_dataset length {len(text_dataset)} and lang_dataset length {len(lang_dataset)} must be the same!"
          return list(zip(text_dataset, lang_dataset))
      else:
          return [[text] for text in text_dataset]
  
file_path: NeMo/scripts/asr_language_modeling/ngram_lm/kenlm_utils.py
imports: import gzip
  import json
  import os
  import numpy
  import torch
  from joblib import Parallel, delayed
  from tqdm.auto import tqdm
  import nemo.collections.asr
  from nemo.collections.asr.parts.submodules.ctc_beam_decoding import DEFAULT_TOKEN_OFFSET
  from nemo.utils import logging
label_desc: Use image node_id 5 for calling a global function with example usage: read_train_file(path, is_aggregate_tokenizer, verbose) and returns: list, unknown