class_docstr: remove overlapping of caption sentences(clip).
  Some statistics:
  caption:
  {'t_clip_len': 246.6448431320854,
  'video_len': 281.09174795676245,
  'clip_tps': 0.8841283727427481,
  'video_tps': 0.7821156477732097,
  'min_clip_len': 0.0,
  'max_clip_len': 398.3,
  'mean_clip_len': 3.196580003006861,
  'num_clip': 77.15897706301081}
  
  raw_caption:
  {'t_clip_len': 238.95908778424115,
  'video_len': 267.5914859862507,
  'clip_tps': 2.4941363624267963,
  'video_tps': 2.258989769647173,
  'min_clip_len': 0.0,
  'max_clip_len': 398.3,
  'mean_clip_len': 3.0537954186814265,
  'num_clip': 78.24986779481756}
class_docstr_tok: ["remove","overlapping","of","caption","sentences","(","clip",")",".","Some","statistics",":","caption",":","{","'t_clip_len","'",":","246.6448431320854",",","'video_len","'",":","281.09174795676245",",","'clip_tps","'",":","0.8841283727427481",",","'video_tps","'",":","0.7821156477732097",",","'min_clip_len","'",":","0.0",",","'max_clip_len","'",":","398.3",",","'mean_clip_len","'",":","3.196580003006861",",","'num_clip","'",":","77.15897706301081","}","raw_caption",":","{","'t_clip_len","'",":","238.95908778424115",",","'video_len","'",":","267.5914859862507",",","'clip_tps","'",":","2.4941363624267963",",","'video_tps","'",":","2.258989769647173",",","'min_clip_len","'",":","0.0",",","'max_clip_len","'",":","398.3",",","'mean_clip_len","'",":","3.0537954186814265",",","'num_clip","'",":","78.24986779481756","}"]
code: def _dedup(self, caption):
      def random_merge(end_idx, start, end, text, starts, ends, texts):
          if random.random() > 0.5:
              # print(clip_idx, "[PARTIAL INTO PREV]", end_idx)
              # overlapped part goes to the end of previous.
              ends[-1] = max(ends[-1], start)  # ?
              rest_text = text[end_idx:].strip()
              if rest_text:
                  starts.append(max(ends[-1], start))
                  ends.append(max(end, starts[-1]))
                  texts.append(rest_text)
          else:  # goes to the beginning of the current.
              # strip the previous.
              left_text = texts[-1][:-end_idx].strip()
              if left_text:
                  # print(clip_idx, "[PREV PARTIAL INTO CUR]", end_idx)
                  ends[-1] = min(ends[-1], start)
                  texts[-1] = left_text
              else:
                  # print(clip_idx, "[PREV LEFT NOTHING ALL INTO CUR]", end_idx)
                  starts.pop(-1)
                  ends.pop(-1)
                  texts.pop(-1)
              starts.append(start)
              ends.append(end)
              texts.append(text)
  
      starts, ends, texts = [], [], []
      for clip_idx, (start, end, text) in enumerate(
          zip(caption["start"], caption["end"], caption["text"])
      ):
          if not isinstance(text, str):
              continue
          text = text.replace("\n", " ").strip()
          if len(text) == 0:
              continue
          starts.append(start)
          ends.append(end)
          texts.append(text)
          break
  
      for clip_idx, (start, end, text) in enumerate(
          zip(
              caption["start"][clip_idx + 1 :],
              caption["end"][clip_idx + 1 :],
              caption["text"][clip_idx + 1 :],
          )
      ):
          if not isinstance(text, str):
              continue
          text = text.replace("\n", " ").strip()
          if len(text) == 0:
              continue
  
          # print(clip_idx, texts[-5:])
          # print(clip_idx, start, end, text)
          if texts[-1].endswith(
              text
          ):  # subset of prev caption -> merge
              # print(clip_idx, "[MERGE INTO PREV]")
              ends[-1] = max(ends[-1], end)
          elif text.startswith(
              texts[-1]
          ):  # superset of prev caption -> merge
              # print(clip_idx, "[PREV MERGE INTO CUR]")
              texts[-1] = text
              starts[-1] = min(starts[-1], start)
              ends[-1] = max(ends[-1], end)
          else:  # overlapping or non-overlapping.
              for end_idx in range(1, len(text) + 1):
                  if texts[-1].endswith(text[:end_idx]):
                      random_merge(
                          end_idx, start, end, text, starts, ends, texts
                      )
                      break
              else:
                  starts.append(start)
                  ends.append(end)
                  texts.append(text)
  
          assert (ends[-1] + 0.001) >= starts[-1] and len(
              texts[-1]
          ) > 0, "{} {} {} <- {} {} {}, {} {} {}".format(
              str(starts[-1]),
              str(ends[-1]),
              texts[-1],
              caption["start"][clip_idx - 1],
              caption["end"][clip_idx - 1],
              caption["text"][clip_idx - 1],
              str(start),
              str(end),
              text,
          )
  
      return {"start": starts, "end": ends, "text": texts}
  
file_path: fairseq/examples/MMPT/mmpt/processors/dedupprocessor.py
imports: import random
  import json
  import pickle
  from tqdm import tqdm
  import os
  import numpy
label_desc: Use image node_id 7 for calling the CaptionDedupProcessor obj's underlying member method code with example usage: obj._dedup(caption) and returns: dict