code: def scan_file(original_text, current_index, tsv_file):
      relabeled_text = read_words(tsv_file)
      # for now, at least, we ignore these markers
      relabeled_indices = [
          idx
          for idx, x in enumerate(relabeled_text)
          if x != "$" and x != "^"
      ]
      relabeled_text = [
          x for x in relabeled_text if x != "$" and x != "^"
      ]
      diffs = SequenceMatcher(
          None, original_text, relabeled_text, False
      )
  
      blocks = diffs.get_matching_blocks()
      assert blocks[-1].size == 0
      if len(blocks) == 1:
          raise ValueError(
              "Could not find a match between %s and the original text"
              % tsv_file
          )
  
      sentences = []
      current_sentence = []
  
      in_mwt = False
      bad_sentence = False
      current_mwt = []
      block_index = 0
      current_block = blocks[0]
      for tsv_index, next_word in enumerate(relabeled_text):
          if not next_word:
              if in_mwt:
                  current_mwt = []
                  in_mwt = False
                  bad_sentence = True
                  print(
                      "Unclosed MWT found at %s line %d"
                      % (tsv_file, tsv_index)
                  )
              if current_sentence:
                  if not bad_sentence:
                      sentences.append(current_sentence)
                  bad_sentence = False
                  current_sentence = []
              continue
  
          # tsv_index will now be inside the current block or before the current block
          while tsv_index >= blocks[block_index].b + current_block.size:
              block_index += 1
              current_block = blocks[block_index]
          # print(tsv_index, current_block.b, current_block.size)
  
          if next_word == "," or next_word == ".":
              # many of these punctuations were added by the relabelers
              current_sentence.append(next_word)
              continue
          if (
              tsv_index >= current_block.b
              and tsv_index < current_block.b + current_block.size
          ):
              # ideal case: in a matching block
              current_sentence.append(next_word)
              continue
  
          # in between blocks... need to handle re-spelled words and MWTs
          if not in_mwt and next_word == "@":
              in_mwt = True
              continue
          if not in_mwt:
              current_sentence.append(next_word)
              continue
          if (
              in_mwt
              and next_word == "@"
              and (
                  tsv_index + 1 < len(relabeled_text)
                  and relabeled_text[tsv_index + 1] == "@"
              )
          ):
              # we'll stop the MWT next time around
              continue
          if in_mwt and next_word == "@":
              if block_index > 0 and (
                  len(current_mwt) == 2 or len(current_mwt) == 3
              ):
                  mwt = "".join(current_mwt)
                  start_original = (
                      blocks[block_index - 1].a
                      + blocks[block_index - 1].size
                  )
                  end_original = current_block.a
                  if find_word(
                      original_text, mwt, start_original, end_original
                  ):
                      current_sentence.append((mwt, current_mwt))
                  else:
                      print(
                          "%d word MWT %s at %s %d.  Should be somewhere in %d %d"
                          % (
                              len(current_mwt),
                              mwt,
                              tsv_file,
                              relabeled_indices[tsv_index],
                              start_original,
                              end_original,
                          )
                      )
                      bad_sentence = True
              elif len(current_mwt) > 6:
                  raise ValueError(
                      "Unreasonably long MWT span in %s at line %d"
                      % (tsv_file, relabeled_indices[tsv_index])
                  )
              elif len(current_mwt) > 3:
                  print(
                      "%d word sequence, stop being lazy - %s %d"
                      % (
                          len(current_mwt),
                          tsv_file,
                          relabeled_indices[tsv_index],
                      )
                  )
                  bad_sentence = True
              else:
                  # short MWT, but it was at the start of a file, and we don't want to search the whole file for the item
                  # TODO, could maybe search the 10 words or so before the start of the block?
                  bad_sentence = True
              current_mwt = []
              in_mwt = False
              continue
          # now we know we are in an MWT... TODO
          current_mwt.append(next_word)
  
      if len(current_sentence) > 0 and not bad_sentence:
          sentences.append(current_sentence)
  
      return current_index, sentences
  
file_path: stanza/stanza/utils/datasets/tokenization/convert_ml_cochin.py
imports: from difflib import SequenceMatcher
  import os
  import random
  import sys
  import stanza.utils.default_paths
label_desc: Use image node_id 5 for calling a global function with example usage: scan_file(original_text, current_index, tsv_file) and returns: current_index,  sentences