code: def __build_dataset_from_text(
      texts: str, lower_case: bool, chunk_size: int
  ):
      if "," in texts:
          texts = texts.split(",")
      else:
          texts = [texts]
  
      num_lines = 0
      text_dataset = []
  
      for text in texts:
          with open(text, "r", encoding="utf-8") as in_reader:
              reader = tqdm(
                  iter(lambda: in_reader.readline(), ""),
                  desc="Read 0 lines",
                  unit=" lines",
              )
  
              for i, line in enumerate(reader):
                  # Clean text line
                  line = line.replace("\n", "").strip()
  
                  if lower_case:
                      line = line.lower()
  
                  if line:
                      text_dataset.append(line)
  
                      num_lines += 1
                      if num_lines % 100000 == 0:
                          reader.set_description(
                              f"Read {num_lines} lines"
                          )
  
                      if num_lines % chunk_size == 0:
                          yield text_dataset, num_lines
  
                          # Empty cache
                          text_dataset = []
  
              logging.info(f"Finished extracting manifest : {text}")
  
          logging.info(
              "Finished extracting all manifests ! Number of sentences : {}".format(
                  num_lines
              )
          )
  
      if len(text_dataset) != 0:
          yield text_dataset, num_lines
  
file_path: NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py
imports: import argparse
  import glob
  import json
  import logging
  import os
  import tarfile
  import joblib
  import numpy
  from tqdm import tqdm
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
label_desc: Use image node_id 1 for calling a global function with example usage: __build_dataset_from_text(texts, lower_case, chunk_size) without return types