code: def check_train_all(raw_data, directions, all_test_data):
      mess_up_train = {}
      data_sizes = {}
      # raw_data = '~chau/data-bin/MineBART/multilingual_mined_100M/en_XX/et_EE-en_XX/all.{en_XX, et_EE}'
      print(
          f"checking training data againsts # {len(all_test_data)} sentences"
      )
      print(
          f"example test data: ",
          [s for i, s in enumerate(all_test_data.keys()) if i < 10],
      )
      for direction in directions:
          src, tgt = direction.split("-")
          path = f"{raw_data}/en_XX/{direction}/all"
          src_path = f"{path}.{src}"
          tgt_path = f"{path}.{tgt}"
          print(f"checking {src_path} {tgt_path}")
          _, size, overlapped_size_counted_dup = check_train_sentences(
              src_path,
              tgt_path,
              direction,
              all_test_data,
              mess_up_train,
          )
          data_sizes[direction] = (size, overlapped_size_counted_dup)
      return mess_up_train, data_sizes
  
file_path: fairseq/examples/multilingual/data_scripts/check_valid_test_overlaps.py
imports: import os
  import argparse
  import pandas
  import sys
label_desc: Use image node_id 6 for calling a global function with example usage: check_train_all(raw_data, directions, all_test_data) and returns: mess_up_train,  data_sizes