code: def get_ptb_tokenized_phrases(dataset):
      """
      Use the PTB tokenizer to retokenize the phrases
  
      Not clear which is better, "Nov." or "Nov ."
      strictAcronym=true makes it do the latter
      tokenizePerLine=true should make it only pay attention to one line at a time
  
      Phrases will be returned as lists of words rather than one string
      """
      with tempfile.TemporaryDirectory() as tempdir:
          phrase_filename = os.path.join(tempdir, "phrases.txt")
          # phrase_filename = "asdf.txt"
          with open(phrase_filename, "w", encoding="utf-8") as fout:
              for item in dataset:
                  # extra newlines are so the tokenizer treats the lines
                  # as separate sentences
                  fout.write("%s\n\n\n" % (item.text))
          tok_filename = os.path.join(tempdir, "tokenized.txt")
          os.system(
              'java edu.stanford.nlp.process.PTBTokenizer -options "strictAcronym=true,tokenizePerLine=true" -preserveLines %s > %s'
              % (phrase_filename, tok_filename)
          )
          with open(tok_filename, encoding="utf-8") as fin:
              tokenized = fin.readlines()
  
      tokenized = [x.strip() for x in tokenized]
      tokenized = [x for x in tokenized if x]
      phrases = [
          SentimentDatum(x.sentiment, y.split())
          for x, y in zip(dataset, tokenized)
      ]
      return phrases
  
file_path: stanza/stanza/utils/datasets/sentiment/process_utils.py
imports: import csv
  import glob
  import json
  import os
  import tempfile
  from collections import namedtuple
  from tqdm import tqdm
  import stanza
  from stanza.models.classifiers.data import SentimentDatum
label_desc: Use image node_id 5 for calling a global function with example usage: get_ptb_tokenized_phrases(dataset) and returns: phrases