class_docstr: remove overlapping of caption sentences(clip).
  Some statistics:
  caption:
  {'t_clip_len': 246.6448431320854,
  'video_len': 281.09174795676245,
  'clip_tps': 0.8841283727427481,
  'video_tps': 0.7821156477732097,
  'min_clip_len': 0.0,
  'max_clip_len': 398.3,
  'mean_clip_len': 3.196580003006861,
  'num_clip': 77.15897706301081}
  
  raw_caption:
  {'t_clip_len': 238.95908778424115,
  'video_len': 267.5914859862507,
  'clip_tps': 2.4941363624267963,
  'video_tps': 2.258989769647173,
  'min_clip_len': 0.0,
  'max_clip_len': 398.3,
  'mean_clip_len': 3.0537954186814265,
  'num_clip': 78.24986779481756}
class_docstr_tok: ["remove","overlapping","of","caption","sentences","(","clip",")",".","Some","statistics",":","caption",":","{","'t_clip_len","'",":","246.6448431320854",",","'video_len","'",":","281.09174795676245",",","'clip_tps","'",":","0.8841283727427481",",","'video_tps","'",":","0.7821156477732097",",","'min_clip_len","'",":","0.0",",","'max_clip_len","'",":","398.3",",","'mean_clip_len","'",":","3.196580003006861",",","'num_clip","'",":","77.15897706301081","}","raw_caption",":","{","'t_clip_len","'",":","238.95908778424115",",","'video_len","'",":","267.5914859862507",",","'clip_tps","'",":","2.4941363624267963",",","'video_tps","'",":","2.258989769647173",",","'min_clip_len","'",":","0.0",",","'max_clip_len","'",":","398.3",",","'mean_clip_len","'",":","3.0537954186814265",",","'num_clip","'",":","78.24986779481756","}"]
code: def save_stat(self, video_id, caption):
      video_fn = os.path.join(
          "data/feat/feat_how2_s3d", video_id + ".npy"
      )
      if os.path.isfile(video_fn):
          with open(
              video_fn, "rb", 1
          ) as fr:  # 24 is the buffer size. buffered
              version = np.lib.format.read_magic(fr)
              shape, fortran, dtype = np.lib.format._read_array_header(
                  fr, version
              )
              video_len = shape[0]
  
          t_clip_len = 0.0
          t_tokens = 0
          for idx, (start, end, text) in enumerate(
              zip(caption["start"], caption["end"], caption["text"])
          ):
              clip_len = (
                  (end - max(caption["end"][idx - 1], start))
                  if idx > 0
                  else end - start
              )
              t_clip_len += clip_len
              t_tokens += len(text.split(" "))
              self.stat["clip_len"].append(clip_len)
          self.stat["t_clip_len"].append(t_clip_len)
          self.stat["video_len"].append(video_len)
          self.stat["clip_tps"].append(t_tokens / t_clip_len)
          self.stat["video_tps"].append(t_tokens / video_len)
  
file_path: fairseq/examples/MMPT/mmpt/processors/dedupprocessor.py
imports: import random
  import json
  import pickle
  from tqdm import tqdm
  import os
  import numpy
label_desc: Use image node_id 5 for calling the CaptionDedupProcessor obj's underlying member method code with example usage: obj.save_stat(video_id, caption) without return types