code: def read_data(input_dir):
      # data for test sets
      test_files = [os.path.join(input_dir, "TEST_100K_ANS.txt")]
      print(test_files)
  
      # data for train and dev sets
      subdirs = [
          os.path.join(input_dir, "article"),
          os.path.join(input_dir, "encyclopedia"),
          os.path.join(input_dir, "news"),
          os.path.join(input_dir, "novel"),
      ]
      files = []
      for subdir in subdirs:
          if not os.path.exists(subdir):
              raise FileNotFoundError(
                  "Expected a directory that did not exist: {}".format(
                      subdir
                  )
              )
          files.extend(glob.glob(os.path.join(subdir, "*.txt")))
  
      test_documents = []
      for filename in test_files:
          print("File name:", filename)
          with open(filename) as fin:
              processed_lines = []
              for line in fin.readlines():
                  line = clean_line(line)
                  words = line.split("|")
                  words = [clean_word(x) for x in words]
                  for word in words:
                      if len(word) > 1 and word[0] == "<":
                          raise ValueError(
                              "Unexpected word '{}' in document {}".format(
                                  word, filename
                              )
                          )
                  words = [x for x in words if x]
                  processed_lines.append(words)
  
              processed_lines = reprocess_lines(processed_lines)
              paragraphs = convert_processed_lines(processed_lines)
  
              test_documents.extend(paragraphs)
      print("Test document finished.")
  
      documents = []
  
      for filename in files:
          with open(filename) as fin:
              print("File:", filename)
              processed_lines = []
              for line in fin.readlines():
                  line = clean_line(line)
                  words = line.split("|")
                  words = [clean_word(x) for x in words]
                  for word in words:
                      if len(word) > 1 and word[0] == "<":
                          raise ValueError(
                              "Unexpected word '{}' in document {}".format(
                                  word, filename
                              )
                          )
                  words = [x for x in words if x]
                  processed_lines.append(words)
  
              processed_lines = reprocess_lines(processed_lines)
              paragraphs = convert_processed_lines(processed_lines)
  
              documents.extend(paragraphs)
  
      print("All documents finished.")
  
      return documents, test_documents
  
file_path: stanza/stanza/utils/datasets/tokenization/convert_th_best.py
imports: import glob
  import os
  import random
  import re
  import sys
  from stanza.utils.datasets.tokenization.process_thai_tokenization import reprocess_lines, write_dataset, convert_processed_lines, write_dataset_best, write_dataset
label_desc: Use image node_id 3 for calling a global function with example usage: read_data(input_dir) and returns: documents,  test_documents