code: def remove_messed_up_sentences(
      raw_data,
      direction,
      mess_up_train,
      mess_up_train_pairs,
      corrected_langs,
  ):
      split = "train"
      src_lang, tgt_lang = direction.split("-")
  
      tgt = f"{raw_data}/{split}.{direction}.{tgt_lang}"
      src = f"{raw_data}/{split}.{direction}.{src_lang}"
      print(f"working on {direction}: ", src, tgt)
      if not os.path.exists(tgt) or not os.path.exists(src):
          return
  
      corrected_tgt = f"{to_folder}/{split}.{direction}.{tgt_lang}"
      corrected_src = f"{to_folder}/{split}.{direction}.{src_lang}"
      line_num = 0
      keep_num = 0
      with open(
          src,
          encoding="utf8",
      ) as fsrc, open(
          tgt,
          encoding="utf8",
      ) as ftgt, open(
          corrected_src, "w", encoding="utf8"
      ) as fsrc_corrected, open(
          corrected_tgt, "w", encoding="utf8"
      ) as ftgt_corrected:
          for s, t in zip(fsrc, ftgt):
              s = s.strip()
              t = t.strip()
              if (
                  t not in mess_up_train
                  and s not in mess_up_train
                  and (s, t) not in mess_up_train_pairs
                  and (t, s) not in mess_up_train_pairs
              ):
                  corrected_langs.add(direction)
                  print(s, file=fsrc_corrected)
                  print(t, file=ftgt_corrected)
                  keep_num += 1
              line_num += 1
              if line_num % 1000 == 0:
                  print(f"completed {line_num} lines", end="\r")
      return line_num, keep_num
  
file_path: fairseq/examples/multilingual/data_scripts/remove_valid_test_in_train.py
imports: import os, sys
  import glob, itertools
  import pandas
label_desc: Use image node_id 9 for calling a global function with example usage: remove_messed_up_sentences(raw_data, direction, mess_up_train, mess_up_train_pairs, corrected_langs) and returns: line_num,  keep_num