code: def process_raw_file(text_file, token_file, sentence_file):
      """
      Process a text file separated into a list of tokens using match_tokens_with_text from the tokenizer
  
      The tokens are one per line in the token_file
      The tokens in the token_file must add up to the text_file modulo whitespace.
  
      Sentences are also one per line in the sentence_file
      These must also add up to text_file
  
      The return format is a list of list of conllu lines representing the sentences.
      The only fields set will be the token index, the token text, and possibly SpaceAfter=No
      where SpaceAfter=No is true if the next token started with no whitespace in the text file
      """
      with open(text_file, encoding="utf-8") as fin:
          text = fin.read()
  
      tokens = read_tokens_file(token_file)
      tokens = [[token for sentence in tokens for token in sentence]]
      tokens_doc = match_tokens_with_text(tokens, text)
  
      assert len(tokens_doc.sentences) == 1
      assert len(tokens_doc.sentences[0].tokens) == len(tokens[0])
  
      sentences = read_sentences_file(sentence_file)
      sentences_doc = match_tokens_with_text([sentences], text)
  
      assert len(sentences_doc.sentences) == 1
      assert len(sentences_doc.sentences[0].tokens) == len(sentences)
  
      start_token_idx = 0
      sentences = []
      for sent_idx, sentence in enumerate(
          sentences_doc.sentences[0].tokens
      ):
          tokens = []
          tokens.append("# sent_id = %d" % (sent_idx + 1))
          tokens.append(
              "# text = %s"
              % text[sentence.start_char : sentence.end_char].replace(
                  "\n", " "
              )
          )
          token_idx = 0
          while token_idx + start_token_idx < len(
              tokens_doc.sentences[0].tokens
          ):
              token = tokens_doc.sentences[0].tokens[
                  token_idx + start_token_idx
              ]
              if token.start_char >= sentence.end_char:
                  # have reached the end of this sentence
                  # continue with the next sentence
                  start_token_idx += token_idx
                  break
  
              if (
                  token_idx + start_token_idx
                  == len(tokens_doc.sentences[0].tokens) - 1
              ):
                  # definitely the end of the document
                  space_after = True
              elif (
                  token.end_char
                  == tokens_doc.sentences[0]
                  .tokens[token_idx + start_token_idx + 1]
                  .start_char
              ):
                  space_after = False
              else:
                  space_after = True
              token = (
                  [str(token_idx + 1), token.text]
                  + ["_"] * 7
                  + ["_" if space_after else "SpaceAfter=No"]
              )
              assert len(token) == 10, "Token length: %d" % len(token)
              token = "\t".join(token)
              tokens.append(token)
              token_idx += 1
          sentences.append(tokens)
      return sentences
  
file_path: stanza/stanza/utils/datasets/tokenization/convert_text_files.py
imports: import argparse
  import os
  import random
  from stanza.models.tokenization.utils import match_tokens_with_text
  import stanza.utils.datasets.common
label_desc: Use image node_id 3 for calling a global function with example usage: process_raw_file(text_file, token_file, sentence_file) and returns: sentences