unilm/edgelm/examples/MMPT/mmpt/processors/dsprocessor.py

# Copyright (c) Facebook, Inc. All Rights Reserved

"""
Processors for all downstream (ds) tasks.
"""

import json
import os
import pickle
import random
import math
import numpy as np
import torch

from collections import defaultdict

from .processor import (
    MetaProcessor,
    VideoProcessor,
    TextProcessor,
    Aligner,
    MMAttentionMask2DProcessor,
)

from .how2processor import TextGenerationProcessor


# ------------- A General Aligner for all downstream tasks-----------------


class DSAligner(Aligner):
    """
    Downstream (DS) aligner shared by all datasets.
    """

    def __call__(self, video_id, video_feature, text_feature, wps=0.7):
        # random sample a starting sec for video.
        video_start = 0
        video_end = min(len(video_feature), self.max_video_len)
        # the whole sequence is a single clip.
        video_clips = {"start": [video_start], "end": [video_end]}

        text_feature = {
            "cap": [text_feature],
            "start": [video_start],
            "end": [len(text_feature) / wps],
        }
        text_clip_indexs = [0]

        vfeats, vmasks = self._build_video_seq(
            video_feature, video_clips
        )
        caps, cmasks = self._build_text_seq(
            text_feature, text_clip_indexs
        )

        return {
            "caps": caps,
            "cmasks": cmasks,
            "vfeats": vfeats,
            "vmasks": vmasks,
            "video_id": video_id,
        }


class NLGTextProcessor(TextProcessor):
    """
    Also return the original text as ref.
    """
    def __call__(self, text_id):
        return super().__call__(text_id), text_id


class DSNLGAligner(DSAligner):
    """extend with the capability of 2d mask for generation."""
    def __init__(self, config):
        super().__init__(config)
        self.attnmasker = MMAttentionMask2DProcessor()
        from transformers import AutoTokenizer
        tokenizer = AutoTokenizer.from_pretrained(
            self.bert_name, use_fast=self.use_fast,
            bos_token="[CLS]", eos_token="[SEP]"
        )
        self.tokenizer = tokenizer
        self.bos_token_id = tokenizer.bos_token_id
        self.eos_token_id = tokenizer.eos_token_id
        self.textgen = TextGenerationProcessor(tokenizer)

    def __call__(self, video_id, video_feature, text_feature):
        output = super().__call__(video_id, video_feature, text_feature[0])
        if self.split == "test":
            # output.update({"ref": text_feature[1]})
            output.update({"ref": self.tokenizer.decode(
                output["caps"], skip_special_tokens=True)})
            text_label = output["caps"]
            cmasks = torch.BoolTensor([1] * text_label.size(0))
            caps = torch.LongTensor([
                self.cls_token_id,
                self.sep_token_id,
                self.bos_token_id])
        else:
            caps, text_label = self.textgen(output["caps"])
            cmasks = output["cmasks"]

        attention_mask = self.attnmasker(
            output["vmasks"], cmasks, "textgen")

        output.update({
            "caps": caps,
            "cmasks": cmasks,
            "text_label": text_label,
            "attention_mask": attention_mask,
        })
        return output


# -------------------- MSRVTT ------------------------


class MSRVTTMetaProcessor(MetaProcessor):
    """MSRVTT dataset.
    reference: `howto100m/msrvtt_dataloader.py`
    """

    def __init__(self, config):
        super().__init__(config)
        import pandas as pd
        data = pd.read_csv(self._get_split_path(config))
        # TODO: add a text1ka flag.
        if config.split == "train" \
                and config.full_test_path is not None \
                and config.jsfusion_path is not None:
            # add testing videos from full_test_path not used by jfusion.
            additional_data = pd.read_csv(config.full_test_path)
            jsfusion_data = pd.read_csv(config.jsfusion_path)

            for video_id in additional_data["video_id"]:
                if video_id not in jsfusion_data["video_id"].values:
                    data = data.append(
                        {"video_id": video_id}, ignore_index=True)

        if config.dup is not None and config.split == "train":
            data = data.append([data] * (config.dup - 1), ignore_index=True)
        self.data = data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        """slightly modify with if condition to combine train/test."""
        vid, sentence = None, None
        vid = self.data["video_id"].values[idx]
        if "sentence" in self.data:  # for testing.
            sentence = self.data["sentence"].values[idx]
        else:  # for training.
            sentence = vid
        return vid, sentence


class MSRVTTTextProcessor(TextProcessor):
    """MSRVTT dataset.
    reference: `msrvtt_dataloader.py` `MSRVTT_TrainDataLoader`.
    TODO (huxu): add max_words.
    """

    def __init__(self, config):
        super().__init__(config)
        self.sentences = None
        if config.json_path is not None and config.split == "train":
            with open(config.json_path) as fd:
                self.data = json.load(fd)
            self.sentences = defaultdict(list)
            for s in self.data["sentences"]:
                self.sentences[s["video_id"]].append(s["caption"])

    def __call__(self, text_id):
        if self.sentences is not None:
            rind = random.randint(0, len(self.sentences[text_id]) - 1)
            sentence = self.sentences[text_id][rind]
        else:
            sentence = text_id
        caption = self.tokenizer(sentence, add_special_tokens=False)
        return caption["input_ids"]


class MSRVTTNLGTextProcessor(MSRVTTTextProcessor):
    """TODO: change dsaligner and merge to avoid any NLG text processor."""
    def __call__(self, text_id):
        if self.sentences is not None:
            rind = random.randint(0, len(self.sentences[text_id]) - 1)
            sentence = self.sentences[text_id][rind]
        else:
            sentence = text_id
        caption = self.tokenizer(sentence, add_special_tokens=False)
        return caption["input_ids"], sentence


class MSRVTTQAMetaProcessor(MetaProcessor):
    """MSRVTT-QA: retrieval-based multi-choice QA from JSFusion dataset.
    For simplicity, we use the train retrieval model.
    reference: `https://github.com/yj-yu/lsmdc`
    """

    def __init__(self, config):
        super().__init__(config)
        import pandas as pd
        csv_data = pd.read_csv(self._get_split_path(config), sep="\t")
        data = []
        for video_id, a1, a2, a3, a4, a5, answer in zip(
                csv_data["vid_key"].values,
                csv_data["a1"].values,
                csv_data["a2"].values,
                csv_data["a3"].values,
                csv_data["a4"].values,
                csv_data["a5"].values,
                csv_data["answer"].values):
            video_id = video_id.replace("msr", "video")
            data.append((video_id, (answer, [a1, a2, a3, a4, a5])))
        self.data = data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]


class MSRVTTQATextProcessor(TextProcessor):
    """MSRVTT-QA dataset.
    text_ans is of format `(answer, [a1, a2, a3, a4, a5])`.
    """

    def __call__(self, text_ans):
        for ans_idx, ans in enumerate(text_ans[1]):
            if isinstance(ans, str):
                text_ans[1][ans_idx] = self.tokenizer(ans, add_special_tokens=False)["input_ids"]
        return text_ans


class MSRVTTQAAligner(DSAligner):
    """MSRVTT dataset.
    similar to sample in how2.
    we call __call__ multiple times.
    """

    def __call__(self, video_id, video_feature, text_feature, wps=0.7):
        caps = []
        cmasks = []
        answer = text_feature[0]
        for ans_idx, _text_feature in enumerate(text_feature[1]):
            output = super().__call__(
                video_id, video_feature, _text_feature, wps)
            caps.append(output["caps"])
            cmasks.append(output["cmasks"])
        output.update({
            "caps": torch.stack(caps),
            "cmasks": torch.stack(cmasks),
            "answers": torch.LongTensor([answer]),
        })
        return output


# -------------------- Youcook -----------------------


class YoucookMetaProcessor(MetaProcessor):
    """Youcook dataset.
    reference: `howto100m/youcook_dataloader.py`
    note that the data can be different as the
    (1) some videos already in Howto100m are removed.
    (2) stop words are removed from caption
    TODO (huxu): make a flag to load the original caption.
    (see youcookii_annotations_trainval.json).

    The max_video_len can be 264 and text can be 64 tokens.
    In reality we may not need that long. see projects/task/youcook.yaml
    """

    def __init__(self, config):
        super().__init__(config)
        vfeat_dir = config.vfeat_dir
        print(self._get_split_path(config))
        with open(self._get_split_path(config), "rb") as fd:
            data = pickle.load(fd)
            all_valid_video_ids = set(
                [os.path.splitext(fn)[0] for fn in os.listdir(vfeat_dir)]
            )
            recs = []
            video_ids = set()
            valid_video_ids = set()
            for rec in data:  # filter videos not available.
                udl_idx = rec["id"].rindex("_")
                video_id = rec["id"][:udl_idx]
                video_ids.add(video_id)
                if video_id in all_valid_video_ids:
                    valid_video_ids.add(video_id)
                    recs.append(rec)
            print("total video_ids in .pkl", len(video_ids))
            print("valid video_ids in .pkl", len(valid_video_ids))
            print("please verify {train,val}_list.txt")
            data = recs
            self.data = data

        with open(config.trainval_annotation) as fd:
            self.youcook_annotation = json.load(fd)["database"]
        if config.use_annotation_text is True:
            print("using text in annotation.")
            self.use_annotation_caption = True
        else:
            self.use_annotation_caption = False

    def __getitem__(self, idx):
        def _get_video_and_caption(rec):
            vid = rec["id"]
            udl_idx = vid.rindex("_")
            video_id, clip_id = vid[:udl_idx], int(vid[udl_idx + 1:])
            clip = self.youcook_annotation[video_id]["annotations"][clip_id]
            start, end = clip["segment"]
            if self.use_annotation_caption:
                caption = clip["sentence"]
            else:
                caption = rec["caption"]
            return (video_id, start, end), caption

        rec = self.data[idx]
        video_info, text_info = _get_video_and_caption(rec)
        return video_info, text_info


class YoucookVideoProcessor(VideoProcessor):
    """video_fn is a tuple of (video_id, start, end) now."""

    def __call__(self, video_fn):
        video_id, start, end = video_fn
        feat = np.load(os.path.join(self.vfeat_dir, video_id + ".npy"))
        return feat[start:end]


class YoucookNLGMetaProcessor(MetaProcessor):
    """NLG uses the original split:
    `train_list.txt` and `val_list.txt`
    """

    def __init__(self, config):
        super().__init__(config)
        vfeat_dir = config.vfeat_dir
        print(self._get_split_path(config))
        with open(self._get_split_path(config)) as fd:
            video_ids = [
                line.strip().split("/")[1] for line in fd.readlines()]
            print("total video_ids in train/val_list.txt", len(video_ids))

            all_valid_video_ids = set(
                [os.path.splitext(fn)[0] for fn in os.listdir(vfeat_dir)]
            )
            video_ids = [
                video_id for video_id in video_ids
                if video_id in all_valid_video_ids]

            print("valid video_ids in train/val_list.txt", len(video_ids))
        with open(config.trainval_annotation) as fd:
            self.youcook_annotation = json.load(fd)["database"]

        data = []
        for video_id in video_ids:
            for clip in self.youcook_annotation[video_id]["annotations"]:
                start, end = clip["segment"]
                caption = clip["sentence"]
                data.append(((video_id, start, end), caption))
        self.data = data

    def __getitem__(self, idx):
        return self.data[idx]


# --------------------- CrossTask -------------------------

class CrossTaskMetaProcessor(MetaProcessor):
    def __init__(self, config):
        super().__init__(config)
        np.random.seed(0)  # deterministic random split.
        task_vids = self._get_vids(
            config.train_csv_path,
            config.vfeat_dir,
            config.annotation_path)

        val_vids = self._get_vids(
            config.val_csv_path,
            config.vfeat_dir,
            config.annotation_path)

        # filter out those task and vids appear in val_vids.
        task_vids = {
            task: [
                vid for vid in vids
                if task not in val_vids or vid not in val_vids[task]]
            for task, vids in task_vids.items()}

        primary_info = self._read_task_info(config.primary_path)
        test_tasks = set(primary_info['steps'].keys())

        # if args.use_related:
        related_info = self._read_task_info(config.related_path)
        task_steps = {**primary_info['steps'], **related_info['steps']}
        n_steps = {**primary_info['n_steps'], **related_info['n_steps']}
        # else:
        #     task_steps = primary_info['steps']
        #     n_steps = primary_info['n_steps']
        all_tasks = set(n_steps.keys())
        # filter and keep task in primary or related.
        task_vids = {
            task: vids for task, vids in task_vids.items()
            if task in all_tasks}
        # vocab-by-step matrix (A) and vocab (M)
        # (huxu): we do not use BoW.
        # A, M = self._get_A(task_steps, share="words")

        train_vids, test_vids = self._random_split(
            task_vids, test_tasks, config.n_train)
        print("train_num_videos", sum(len(vids) for vids in train_vids.values()))
        print("test_num_videos", sum(len(vids) for vids in test_vids.values()))
        # added by huxu to automatically determine the split.
        split_map = {
            "train": train_vids,
            "valid": test_vids,
            "test": test_vids
        }
        task_vids = split_map[config.split]

        self.vids = []
        for task, vids in task_vids.items():
            self.vids.extend([(task, vid) for vid in vids])
        self.task_steps = task_steps
        self.n_steps = n_steps

    def __getitem__(self, idx):
        task, vid = self.vids[idx]
        n_steps = self.n_steps[task]
        steps = self.task_steps[task]
        assert len(steps) == n_steps
        return (task, vid, steps, n_steps), (task, vid, steps, n_steps)

    def __len__(self):
        return len(self.vids)

    def _random_split(self, task_vids, test_tasks, n_train):
        train_vids = {}
        test_vids = {}
        for task, vids in task_vids.items():
            if task in test_tasks and len(vids) > n_train:
                train_vids[task] = np.random.choice(
                    vids, n_train, replace=False).tolist()
                test_vids[task] = [
                    vid for vid in vids if vid not in train_vids[task]]
            else:
                train_vids[task] = vids
        return train_vids, test_vids

    def _get_vids(self, path, vfeat_dir, annotation_path):
        """refactored from
        https://github.com/DmZhukov/CrossTask/blob/master/data.py
        changes: add `vfeat_dir` to check if the video is available.
        add `annotation_path` to check if the video is available.
        """

        task_vids = {}
        with open(path, 'r') as f:
            for line in f:
                task, vid, url = line.strip().split(',')
                # double check the video is available.
                if not os.path.exists(
                        os.path.join(vfeat_dir, vid + ".npy")):
                    continue
                # double check the annotation is available.
                if not os.path.exists(os.path.join(
                        annotation_path,
                        task + "_" + vid + ".csv")):
                    continue
                if task not in task_vids:
                    task_vids[task] = []
                task_vids[task].append(vid)
        return task_vids

    def _read_task_info(self, path):
        titles = {}
        urls = {}
        n_steps = {}
        steps = {}
        with open(path, 'r') as f:
            idx = f.readline()
            while idx != '':
                idx = idx.strip()
                titles[idx] = f.readline().strip()
                urls[idx] = f.readline().strip()
                n_steps[idx] = int(f.readline().strip())
                steps[idx] = f.readline().strip().split(',')
                next(f)
                idx = f.readline()
        return {
            'title': titles,
            'url': urls,
            'n_steps': n_steps,
            'steps': steps
        }

    def _get_A(self, task_steps, share="words"):
        raise ValueError("running get_A is not allowed for BERT.")
        """Step-to-component matrices."""
        if share == 'words':
            # share words
            task_step_comps = {
                task: [step.split(' ') for step in steps]
                for task, steps in task_steps.items()}
        elif share == 'task_words':
            # share words within same task
            task_step_comps = {
                task: [[task+'_'+tok for tok in step.split(' ')] for step in steps]
                for task, steps in task_steps.items()}
        elif share == 'steps':
            # share whole step descriptions
            task_step_comps = {
                task: [[step] for step in steps] for task, steps in task_steps.items()}
        else:
            # no sharing
            task_step_comps = {
                task: [[task+'_'+step] for step in steps]
                for task, steps in task_steps.items()}
        # BERT tokenizer here?
        vocab = []
        for task, steps in task_step_comps.items():
            for step in steps:
                vocab.extend(step)
        vocab = {comp: m for m, comp in enumerate(set(vocab))}
        M = len(vocab)
        A = {}
        for task, steps in task_step_comps.items():
            K = len(steps)
            a = torch.zeros(M, K)
            for k, step in enumerate(steps):
                a[[vocab[comp] for comp in step], k] = 1
            a /= a.sum(dim=0)
            A[task] = a
        return A, M


class CrossTaskVideoProcessor(VideoProcessor):
    def __call__(self, video_fn):
        task, vid, steps, n_steps = video_fn
        video_fn = os.path.join(self.vfeat_dir, vid + ".npy")
        feat = np.load(video_fn)
        return feat


class CrossTaskTextProcessor(TextProcessor):
    def __call__(self, text_id):
        task, vid, steps, n_steps = text_id
        step_ids = []
        for step_str in steps:
            step_ids.append(
                self.tokenizer(step_str, add_special_tokens=False)["input_ids"]
            )
        return step_ids


class CrossTaskAligner(Aligner):
    """
    TODO: it's not clear yet the formulation of the task; finish this later.
    """
    def __init__(self, config):
        super().__init__(config)
        self.annotation_path = config.annotation_path
        self.sliding_window = config.sliding_window
        self.sliding_window_size = config.sliding_window_size

    def __call__(self, video_id, video_feature, text_feature):
        task, vid, steps, n_steps = video_id
        annot_path = os.path.join(
            self.annotation_path, task + '_' + vid + '.csv')
        video_len = len(video_feature)

        labels = torch.from_numpy(self._read_assignment(
            video_len, n_steps, annot_path)).float()

        vfeats, vmasks, targets = [], [], []
        # sliding window on video features and targets.
        for window_start in range(0, video_len, self.sliding_window):
            video_start = 0
            video_end = min(video_len - window_start, self.sliding_window_size)
            video_clip = {"start": [video_start], "end": [video_end]}

            vfeat, vmask = self._build_video_seq(
                video_feature[window_start: window_start + video_end],
                video_clip
            )

            target = labels[window_start: window_start + video_end]
            assert len(vfeat) >= len(target), "{},{}".format(len(vfeat), len(target))
            # TODO: randomly drop all zero targets for training ?
            # if self.split == "train" and target.sum() == 0:
            #     continue
            vfeats.append(vfeat)
            vmasks.append(vmask)
            targets.append(target)

            if (video_len - window_start) <= self.sliding_window_size:
                break

        vfeats = torch.stack(vfeats)
        vmasks = torch.stack(vmasks)
        targets = torch.cat(targets, dim=0)

        caps, cmasks = [], []
        for step in text_feature:
            step_text_feature = {"start": [0], "end": [1], "cap": [step]}
            step_text_clip_index = [0]
            cap, cmask = self._build_text_seq(
                step_text_feature, step_text_clip_index
            )
            caps.append(cap)
            cmasks.append(cmask)
        caps = torch.stack(caps)
        cmasks = torch.stack(cmasks)

        return {
            "caps": caps,
            "cmasks": cmasks,
            "vfeats": vfeats,  # X for original code.
            "vmasks": vmasks,
            "targets": targets,
            "video_id": vid,
            "task": task,
            "video_len": video_len  # for later checking.
        }

    def _read_assignment(self, T, K, path):
        """
        refactored from https://github.com/DmZhukov/CrossTask/blob/master/data.py
        Howto interpret contraints on loss that is going to be minimized:
        lambd is a big number;
        self.lambd * C is a big number for all valid position (csv stores invalids)

        def forward(self, O, Y, C):
            return (Y*(self.lambd * C - self.lsm(O))).mean(dim=0).sum()

        This will load the csv file and fill-in the step col from start to end rows.
        """

        Y = np.zeros([T, K], dtype=np.uint8)
        with open(path, 'r') as f:
            for line in f:
                step, start, end = line.strip().split(',')
                start = int(math.floor(float(start)))
                end = int(math.ceil(float(end)))
                step = int(step) - 1
                Y[start:end, step] = 1
        return Y


# --------------------- COIN -------------------------

class MetaTextBinarizer(Aligner):
    def __call__(self, text_feature):
        text_feature = {
            "cap": [text_feature],
            "start": [0.],
            "end": [100.],
        }
        text_clip_indexs = [0]

        caps, cmasks = self._build_text_seq(
            text_feature, text_clip_indexs
        )
        return {"caps": caps, "cmasks": cmasks}


class COINActionSegmentationMetaProcessor(MetaProcessor):
    split_map = {
        "train": "training",
        "valid": "testing",
        "test": "testing",
    }

    def __init__(self, config):
        super().__init__(config)
        with open(self._get_split_path(config)) as fr:
            database = json.load(fr)["database"]
        id2label = {}
        data = []
        # filter the data by split.
        for video_id, rec in database.items():
            # always use testing to determine label_set
            if rec["subset"] == "testing":
                for segment in rec["annotation"]:
                    id2label[int(segment["id"])] = segment["label"]
        # text_labels is used for ZS setting
        self.text_labels = ["none"] * len(id2label)
        for label_id in id2label:
            self.text_labels[label_id-1] = id2label[label_id]

        id2label[0] = "O"
        print("num of labels", len(id2label))

        for video_id, rec in database.items():
            if not os.path.isfile(os.path.join(config.vfeat_dir, video_id + ".npy")):
                continue
            if rec["subset"] == COINActionSegmentationMetaProcessor.split_map[self.split]:
                starts, ends, labels = [], [], []
                for segment in rec["annotation"]:
                    start, end = segment["segment"]
                    label = int(segment["id"])
                    starts.append(start)
                    ends.append(end)
                    labels.append(label)
                data.append(
                    (video_id, {"start": starts, "end": ends, "label": labels}))
        self.data = data

    def meta_text_labels(self, config):
        from transformers import default_data_collator
        from ..utils import get_local_rank

        text_processor = TextProcessor(config)
        binarizer = MetaTextBinarizer(config)
        # TODO: add prompts to .yaml.
        text_labels = [label for label in self.text_labels]

        if get_local_rank() == 0:
            print(text_labels)

        outputs = []
        for text_label in text_labels:
            text_feature = text_processor(text_label)
            outputs.append(binarizer(text_feature))
        return default_data_collator(outputs)

    def __getitem__(self, idx):
        return self.data[idx]


class COINActionSegmentationTextProcessor(TextProcessor):
    def __call__(self, text_label):
        return text_label


class COINActionSegmentationAligner(Aligner):
    def __init__(self, config):
        super().__init__(config)
        self.sliding_window = config.sliding_window
        self.sliding_window_size = config.sliding_window_size

    def __call__(self, video_id, video_feature, text_feature):
        starts, ends, label_ids = text_feature["start"], text_feature["end"], text_feature["label"]
        # sliding window.
        video_len = len(video_feature)

        vfeats, vmasks, targets = [], [], []
        # sliding window on video features and targets.
        for window_start in range(0, video_len, self.sliding_window):
            video_start = 0
            video_end = min(video_len - window_start, self.sliding_window_size)
            video_clip = {"start": [video_start], "end": [video_end]}
            vfeat, vmask = self._build_video_seq(
                video_feature[window_start: window_start + video_end],
                video_clip
            )
            # covers video length only.
            target = torch.full_like(vmask, -100, dtype=torch.long)
            target[vmask] = 0
            for start, end, label_id in zip(starts, ends, label_ids):
                if (window_start < end) and (start < (window_start + video_end)):
                    start_offset = max(0, math.floor(start) - window_start)
                    end_offset = min(video_end, math.ceil(end) - window_start)
                    target[start_offset:end_offset] = label_id
            vfeats.append(vfeat)
            vmasks.append(vmask)
            targets.append(target)
            if (video_len - window_start) <= self.sliding_window_size:
                break

        vfeats = torch.stack(vfeats)
        vmasks = torch.stack(vmasks)
        targets = torch.stack(targets)
        video_targets = torch.full((video_len,), 0)
        for start, end, label_id in zip(starts, ends, label_ids):
            start_offset = max(0, math.floor(start))
            end_offset = min(video_len, math.ceil(end))
            video_targets[start_offset:end_offset] = label_id

        caps = torch.LongTensor(
            [[self.cls_token_id, self.sep_token_id,
              self.pad_token_id, self.sep_token_id]],
            ).repeat(vfeats.size(0), 1)
        cmasks = torch.BoolTensor(
            [[0, 1, 0, 1]]  # pad are valid for attention.
            ).repeat(vfeats.size(0), 1)
        return {
            "caps": caps,
            "cmasks": cmasks,
            "vfeats": vfeats,  # X for original code.
            "vmasks": vmasks,
            "targets": targets,
            "video_id": video_id,
            "video_len": video_len,  # for later checking.
            "video_targets": video_targets
        }


class DiDeMoMetaProcessor(MetaProcessor):
    """reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
    https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py
    """
    def __init__(self, config):
        super().__init__(config)

        assert "test" in self._get_split_path(config), "DiDeMo only supports zero-shot testing for now."

        with open(self._get_split_path(config)) as data_file:
            json_data = json.load(data_file)

        data = []
        for record in json_data:
            data.append((record["video"], record["description"]))
        self.data = data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]


class DiDeMoTextProcessor(TextProcessor):
    """reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
    https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py
    """

    def __call__(self, text):
        return self.tokenizer(text, add_special_tokens=False)["input_ids"]


class DiDeMoAligner(DSAligner):
    """
    check video length.
    """

    def __call__(self, video_id, video_feature, text_feature):
        # print(video_feature.shape[0])
        return super().__call__(video_id, video_feature, text_feature)