code: def rst_to_outputs(files, min_len=30, max_len=2048 // 2 - 30):
      # account for sequence length (context window) including prompt and input and output
  
      # os.system('pandoc -f rst -t plain ./expert_settings/nlp_settings.rst')
      import pypandoc
  
      basedir = os.path.abspath(os.getcwd())
  
      outputs = []
      for fil in files:
          os.chdir(basedir)
          os.chdir(os.path.dirname(fil))
          fil = os.path.basename(fil)
          print("Processing %s" % fil, flush=True)
          # out_format can be one of: asciidoc, asciidoctor, beamer, biblatex, bibtex, commonmark, commonmark_x,
          # context, csljson, docbook, docbook4, docbook5, docx, dokuwiki,
          # dzslides, epub, epub2, epub3, fb2, gfm, haddock, html, html4, html5, icml,
          # ipynb, jats, jats_archiving, jats_articleauthoring, jats_publishing, jira,
          # json, latex, man,
          # markdown, markdown_github, markdown_mmd, markdown_phpextra, markdown_strict,
          # mediawiki, ms, muse, native, odt, opendocument, opml, org, pdf, plain, pptx,
          # revealjs, rst, rtf, s5, slideous, slidy, tei, texinfo, textile, xwiki, zimwiki
          out_format = "plain"
          # avoid extra new lines injected into text
          extra_args = ["--wrap=preserve", '--resource path="%s" % dst']
  
          plain_list = []
          try:
              # valid for expert settings
              input_rst = pypandoc.convert_file(fil, "rst")
              input_list = input_rst.split("\n``")
              for input_subrst in input_list:
                  input_plain = pypandoc.convert_text(
                      input_subrst, format="rst", to="plain"
                  )
                  plain_list.append([input_plain, fil])
          except Exception as e:
              print("file exception: %s %s" % (fil, str(e)), flush=True)
  
          if not plain_list:
              # if failed to process as pieces of rst, then
              output = pypandoc.convert_file(
                  fil, out_format, extra_args=extra_args, format="rst"
              )
              outputs1 = get_sentences(output, length=max_len)
              for oi, output in enumerate(outputs1):
                  output = output.replace("\n\n", "\n")
                  plain_list.append([output, fil])
          outputs.extend(plain_list)
  
      # report:
      # [print(len(x)) for x in outputs]
  
      # deal with blocks longer than context size (sequence length) of 2048
      new_outputs = []
      num_truncated = 0
      num_orig = len(outputs)
      for output, fil in outputs:
          if len(output) < max_len:
              new_outputs.append([output, fil])
              continue
          outputs1 = get_sentences(output, length=max_len)
          for oi, output1 in enumerate(outputs1):
              output1 = output1.replace("\n\n", "\n")
              new_outputs.append([output1, fil])
          num_truncated += 1
      print(
          "num_orig: %s num_truncated: %s" % (num_orig, num_truncated),
          flush=True,
      )
  
      new_outputs = [
          [k.strip(), fil]
          for k, fil in new_outputs
          if len(k.strip()) > min_len
      ]
  
      return new_outputs
  
imports: import ast
  import concurrent.futures
  import contextlib
  import hashlib
  import json
  import os
  import shutil
  import signal
  import sys
  import traceback
  from concurrent.futures import ProcessPoolExecutor
  import psutil
  import pytest
  import pandas
  import numpy
  from tqdm import tqdm
  from utils import flatten_list, remove
  from joblib import Parallel, delayed, effective_n_jobs
  from sklearn.utils import gen_even_slices
  from sklearn.utils.validation import _num_samples
label_desc: Use image node_id 6 for calling a global function with example usage: rst_to_outputs(files, min_len, max_len) and returns: new_outputs