update

.gitignore

@@ -2,7 +2,7 @@
 *.cache
 *.dev.py
 state_dict/
-
+TAD*/
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -4,7 +4,7 @@ emoji: 🛡️
 colorFrom: gray
 colorTo: green
 sdk: gradio
-sdk_version: 3.0.19
+sdk_version:  3.20.1
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -1,15 +1,13 @@
 import os
-import random
 import zipfile
-from difflib import Differ
 
 import gradio as gr
 import nltk
 import pandas as pd
-from findfile import find_files
+import requests
+from flask import Flask
 
 from anonymous_demo import TADCheckpointManager
-from textattack import Attacker
 from textattack.attack_recipes import (
     BAEGarg2019,
     PWWSRen2019,
@@ -21,60 +19,7 @@ from textattack.attack_recipes import (
     CLARE2020,
 )
 from textattack.attack_results import SuccessfulAttackResult
-from textattack.datasets import Dataset
-from textattack.models.wrappers import HuggingFaceModelWrapper
-
-z = zipfile.ZipFile("checkpoints.zip", "r")
-z.extractall(os.getcwd())
-
-
-class ModelWrapper(HuggingFaceModelWrapper):
-    def __init__(self, model):
-        self.model = model  # pipeline = pipeline
-
-    def __call__(self, text_inputs, **kwargs):
-        outputs = []
-        for text_input in text_inputs:
-            raw_outputs = self.model.infer(text_input, print_result=False, **kwargs)
-            outputs.append(raw_outputs["probs"])
-        return outputs
-
-
-class SentAttacker:
-    def __init__(self, model, recipe_class=BAEGarg2019):
-        model = model
-        model_wrapper = ModelWrapper(model)
-
-        recipe = recipe_class.build(model_wrapper)
-        # WordNet defaults to english. Set the default language to French ('fra')
-
-        # recipe.transformation.language = "en"
-
-        _dataset = [("", 0)]
-        _dataset = Dataset(_dataset)
-
-        self.attacker = Attacker(recipe, _dataset)
-
-
-def diff_texts(text1, text2):
-    d = Differ()
-    return [
-        (token[2:], token[0] if token[0] != " " else None)
-        for token in d.compare(text1, text2)
-    ]
-
-
-def get_ensembled_tad_results(results):
-    target_dict = {}
-    for r in results:
-        target_dict[r["label"]] = (
-            target_dict.get(r["label"]) + 1 if r["label"] in target_dict else 1
-        )
-
-    return dict(zip(target_dict.values(), target_dict.keys()))[
-        max(target_dict.values())
-    ]
-
+from utils import SentAttacker, get_agnews_example, get_sst2_example, get_amazon_example, get_imdb_example, diff_texts
 
 nltk.download("omw-1.4")
 
@@ -89,204 +34,37 @@ attack_recipes = {
     "iga": IGAWang2019,
     "ga": GeneticAlgorithmAlzantot2018,
     "deepwordbug": DeepWordBugGao2018,
-    'clare': CLARE2020,
+    "clare": CLARE2020,
 }
 
-for attacker in ["pwws", "bae", "textfooler", "deepwordbug"]:
-    for dataset in [
-        "agnews10k",
-        "amazon",
-        "sst2",
-        # 'imdb'
-    ]:
-        if "tad-{}".format(dataset) not in tad_classifiers:
-            tad_classifiers[
-                "tad-{}".format(dataset)
-            ] = TADCheckpointManager.get_tad_text_classifier(
-                "tad-{}".format(dataset).upper()
-            )
-
-        sent_attackers["tad-{}{}".format(dataset, attacker)] = SentAttacker(
-            tad_classifiers["tad-{}".format(dataset)], attack_recipes[attacker]
-        )
-        tad_classifiers["tad-{}".format(dataset)].sent_attacker = sent_attackers[
-            "tad-{}pwws".format(dataset)
-        ]
-
-
-def get_sst2_example():
-    filter_key_words = [
-        ".py",
-        ".md",
-        "readme",
-        "log",
-        "result",
-        "zip",
-        ".state_dict",
-        ".model",
-        ".png",
-        "acc_",
-        "f1_",
-        ".origin",
-        ".adv",
-        ".csv",
-    ]
-
-    dataset_file = {"train": [], "test": [], "valid": []}
-    dataset = "sst2"
-    search_path = "./"
-    task = "text_defense"
-    dataset_file["test"] += find_files(
-        search_path,
-        [dataset, "test", task],
-        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
-        + filter_key_words,
-    )
-
-    for dat_type in ["test"]:
-        data = []
-        label_set = set()
-        for data_file in dataset_file[dat_type]:
-            with open(data_file, mode="r", encoding="utf8") as fin:
-                lines = fin.readlines()
-                for line in lines:
-                    text, label = line.split("$LABEL$")
-                    text = text.strip()
-                    label = int(label.strip())
-                    data.append((text, label))
-                    label_set.add(label)
-        return data[random.randint(0, len(data))]
-
-
-def get_agnews_example():
-    filter_key_words = [
-        ".py",
-        ".md",
-        "readme",
-        "log",
-        "result",
-        "zip",
-        ".state_dict",
-        ".model",
-        ".png",
-        "acc_",
-        "f1_",
-        ".origin",
-        ".adv",
-        ".csv",
-    ]
-
-    dataset_file = {"train": [], "test": [], "valid": []}
-    dataset = "agnews"
-    search_path = "./"
-    task = "text_defense"
-    dataset_file["test"] += find_files(
-        search_path,
-        [dataset, "test", task],
-        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
-        + filter_key_words,
-    )
-    for dat_type in ["test"]:
-        data = []
-        label_set = set()
-        for data_file in dataset_file[dat_type]:
-            with open(data_file, mode="r", encoding="utf8") as fin:
-                lines = fin.readlines()
-                for line in lines:
-                    text, label = line.split("$LABEL$")
-                    text = text.strip()
-                    label = int(label.strip())
-                    data.append((text, label))
-                    label_set.add(label)
-        return data[random.randint(0, len(data))]
-
-
-def get_amazon_example():
-    filter_key_words = [
-        ".py",
-        ".md",
-        "readme",
-        "log",
-        "result",
-        "zip",
-        ".state_dict",
-        ".model",
-        ".png",
-        "acc_",
-        "f1_",
-        ".origin",
-        ".adv",
-        ".csv",
-    ]
-
-    dataset_file = {"train": [], "test": [], "valid": []}
-    dataset = "amazon"
-    search_path = "./"
-    task = "text_defense"
-    dataset_file["test"] += find_files(
-        search_path,
-        [dataset, "test", task],
-        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
-        + filter_key_words,
-    )
-
-    for dat_type in ["test"]:
-        data = []
-        label_set = set()
-        for data_file in dataset_file[dat_type]:
-            with open(data_file, mode="r", encoding="utf8") as fin:
-                lines = fin.readlines()
-                for line in lines:
-                    text, label = line.split("$LABEL$")
-                    text = text.strip()
-                    label = int(label.strip())
-                    data.append((text, label))
-                    label_set.add(label)
-        return data[random.randint(0, len(data))]
-
-
-def get_imdb_example():
-    filter_key_words = [
-        ".py",
-        ".md",
-        "readme",
-        "log",
-        "result",
-        "zip",
-        ".state_dict",
-        ".model",
-        ".png",
-        "acc_",
-        "f1_",
-        ".origin",
-        ".adv",
-        ".csv",
-    ]
-
-    dataset_file = {"train": [], "test": [], "valid": []}
-    dataset = "imdb"
-    search_path = "./"
-    task = "text_defense"
-    dataset_file["test"] += find_files(
-        search_path,
-        [dataset, "test", task],
-        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
-        + filter_key_words,
-    )
+app = Flask(__name__)
+
+
+def init():
+    if not os.path.exists("TAD-SST2"):
+        z = zipfile.ZipFile("checkpoints.zip", "r")
+        z.extractall(os.getcwd())
+
+    for attacker in ["pwws", "bae", "textfooler", "deepwordbug"]:
+        for dataset in [
+            "agnews10k",
+            "amazon",
+            "sst2",
+            # 'imdb'
+        ]:
+            if "tad-{}".format(dataset) not in tad_classifiers:
+                tad_classifiers[
+                    "tad-{}".format(dataset)
+                ] = TADCheckpointManager.get_tad_text_classifier(
+                    "tad-{}".format(dataset).upper()
+                )
 
-    for dat_type in ["test"]:
-        data = []
-        label_set = set()
-        for data_file in dataset_file[dat_type]:
-            with open(data_file, mode="r", encoding="utf8") as fin:
-                lines = fin.readlines()
-                for line in lines:
-                    text, label = line.split("$LABEL$")
-                    text = text.strip()
-                    label = int(label.strip())
-                    data.append((text, label))
-                    label_set.add(label)
-        return data[random.randint(0, len(data))]
+            sent_attackers["tad-{}{}".format(dataset, attacker)] = SentAttacker(
+                tad_classifiers["tad-{}".format(dataset)], attack_recipes[attacker]
+            )
+            tad_classifiers["tad-{}".format(dataset)].sent_attacker = sent_attackers[
+                "tad-{}pwws".format(dataset)
+            ]
 
 
 cache = set()
@@ -311,11 +89,11 @@ def generate_adversarial_example(dataset, attacker, text=None, label=None):
     ].attacker.simple_attack(text, int(label))
     if isinstance(attack_result, SuccessfulAttackResult):
         if (
-            attack_result.perturbed_result.output
-            != attack_result.original_result.ground_truth_output
+                attack_result.perturbed_result.output
+                != attack_result.original_result.ground_truth_output
         ) and (
-            attack_result.original_result.output
-            == attack_result.original_result.ground_truth_output
+                attack_result.original_result.output
+                == attack_result.original_result.ground_truth_output
         ):
             # with defense
             result = tad_classifiers["tad-{}".format(dataset.lower())].infer(
@@ -367,133 +145,155 @@ def generate_adversarial_example(dataset, attacker, text=None, label=None):
     )
 
 
-demo = gr.Blocks()
-with demo:
-    gr.Markdown(
-        "# <p align='center'>  Reactive Perturbation Defocusing for Textual Adversarial Defense </p> "
-    )
-
-    gr.Markdown("## <p align='center'>Clarifications</p>")
-    gr.Markdown(
-        "- This demo has no mechanism to ensure the adversarial example will be correctly repaired by RPD."
-        " The repair success rate is actually the performance reported in the paper (approximately up to 97%.)"
-    )
-    gr.Markdown(
-        "- The red (+) and green (-) colors in the character edition indicate the character is added "
-        "or deleted in the adversarial example compared to the original input natural example."
-    )
-    gr.Markdown(
-        "- The adversarial example and repaired adversarial example may be unnatural to read, "
-        "while it is because the attackers usually generate unnatural perturbations."
-        "RPD does not introduce additional unnatural perturbations."
-    )
-    gr.Markdown(
-        "- To our best knowledge, Reactive Perturbation Defocusing is a novel approach in adversarial defense "
-        ". RPD significantly (>10% defense accuracy improvement) outperforms the state-of-the-art methods."
-    )
-    gr.Markdown(
-        "- The DeepWordBug is an unknown attacker to RPD's adversarial detector, which shows the robustness of RPD."
-    )
-
-    gr.Markdown("## <p align='center'>Natural Example Input</p>")
-    with gr.Group():
-        with gr.Row():
-            input_dataset = gr.Radio(
-                choices=["SST2", "AGNews10K", "Amazon"],
-                value="SST2",
-                label="Select a testing dataset and an adversarial attacker to generate an adversarial example.",
-            )
-            input_attacker = gr.Radio(
-                choices=[
-                    "BAE",
-                    "PWWS",
-                    "TextFooler",
-                    "DeepWordBug"
-                ],
-                value="TextFooler",
-                label="Choose an Adversarial Attacker for generating an adversarial example to attack the model.",
-            )
+def run_demo(dataset, attacker, text=None, label=None):
+
+    try:
+        data = {
+            "dataset": dataset,
+            "attacker": attacker,
+            "text": text,
+            "label": label,
+        }
+        response = requests.post('https://rpddemo.pagekite.me/api/generate_adversarial_example', json=data)
+        result = response.json()
+        print(response.json())
+        return (
+            result["text"],
+            result["label"],
+            result["restored_text"],
+            result["result_label"],
+            result["perturbed_text"],
+            result["text_diff"],
+            result["perturbed_diff"],
+            result["restored_diff"],
+            result["output"],
+            pd.DataFrame(result["classification_df"]),
+            pd.DataFrame(result["advdetection_df"]),
+        )
+    except Exception as e:
+        print(e)
+        return generate_adversarial_example(dataset, attacker, text, label)
+
+if __name__ == "__main__":
+
+    init()
+
+    demo = gr.Blocks()
+
+    with demo:
+        gr.Markdown("<h1 align='center'>Reactive Perturbation Defocusing for Textual Adversarial Defense</h1>")
+        gr.Markdown("<h3 align='center'>Clarifications</h2>")
+        gr.Markdown("""
+    - This demo has no mechanism to ensure the adversarial example will be correctly repaired by RPD. The repair success rate is actually the performance reported in the paper (approximately up to 97%).
+    - The adversarial example and repaired adversarial example may be unnatural to read, while it is because the attackers usually generate unnatural perturbations. RPD does not introduce additional unnatural perturbations.
+    - To our best knowledge, Reactive Perturbation Defocusing is a novel approach in adversarial defense. RPD significantly (>10% defense accuracy improvement) outperforms the state-of-the-art methods.
+    - The DeepWordBug is an unknown attacker to the adversarial detector and reactive defense module. DeepWordBug has different attacking patterns from other attackers and shows the generalizability and robustness of RPD.
+    """)
+        gr.Markdown("<h2 align='center'>Natural Example Input</h2>")
         with gr.Group():
             with gr.Row():
-                input_sentence = gr.Textbox(
-                    placeholder="Input a natural example...",
-                    label="Alternatively, input a natural example and its original label to generate an adversarial example.",
+                input_dataset = gr.Radio(
+                    choices=["SST2", "AGNews10K", "Amazon"],
+                    value="Amazon",
+                    label="Select a testing dataset and an adversarial attacker to generate an adversarial example.",
                 )
-                input_label = gr.Textbox(
-                    placeholder="Original label...", label="Original Label"
+                input_attacker = gr.Radio(
+                    choices=["BAE", "PWWS", "TextFooler", "DeepWordBug"],
+                    value="TextFooler",
+                    label="Choose an Adversarial Attacker for generating an adversarial example to attack the model.",
                 )
+            with gr.Group():
+                with gr.Row():
+                    input_sentence = gr.Textbox(
+                        placeholder="Input a natural example...",
+                        label="Alternatively, input a natural example and its original label to generate an adversarial example.",
+                    )
+                    input_label = gr.Textbox(
+                        placeholder="Original label...", label="Original Label"
+                    )
+
+        button_gen = gr.Button(
+            "Generate an adversarial example to repair using RPD (it will takes 1-10 minutes because no GPU is available)",
+            variant="primary",
+        )
 
-    button_gen = gr.Button(
-        "Generate an adversarial example and repair using RPD (No GPU, Time:3-10 mins )",
-        variant="primary",
-    )
+        gr.Markdown("<h2 align='center'>Generated Adversarial Example and Repaired Adversarial Example</h2>")
 
-    gr.Markdown(
-        "## <p align='center'>Generated Adversarial Example and Repaired Adversarial Example</p>"
-    )
-    with gr.Group():
         with gr.Column():
-            with gr.Row():
-                output_original_example = gr.Textbox(label="Original Example")
-                output_original_label = gr.Textbox(label="Original Label")
-            with gr.Row():
-                output_adv_example = gr.Textbox(label="Adversarial Example")
-                output_adv_label = gr.Textbox(label="Perturbed Label")
-            with gr.Row():
-                output_repaired_example = gr.Textbox(
-                    label="Repaired Adversarial Example by RPD"
-                )
-                output_repaired_label = gr.Textbox(label="Repaired Label")
+            with gr.Group():
+                with gr.Row():
+                    output_original_example = gr.Textbox(label="Original Example")
+                    output_original_label = gr.Textbox(label="Original Label")
+                with gr.Row():
+                    output_adv_example = gr.Textbox(label="Adversarial Example")
+                    output_adv_label = gr.Textbox(label="Predicted Label of the Adversarial Example")
+                with gr.Row():
+                    output_repaired_example = gr.Textbox(
+                        label="Repaired Adversarial Example by RPD"
+                    )
+                    output_repaired_label = gr.Textbox(label="Predicted Label of the Repaired Adversarial Example")
+
+        gr.Markdown("<h2 align='center'>Example Difference (Comparisons)</p>")
+        gr.Markdown("""
+    <p align='center'>The (+) and (-) in the boxes indicate the added and deleted characters in the adversarial example compared to the original input natural example.</p>
+        """)
+        ori_text_diff = gr.HighlightedText(
+            label="The Original Natural Example",
+            combine_adjacent=True,
+        )
+        adv_text_diff = gr.HighlightedText(
+            label="Character Editions of Adversarial Example Compared to the Natural Example",
+            combine_adjacent=True,
+        )
+        restored_text_diff = gr.HighlightedText(
+            label="Character Editions of Repaired Adversarial Example Compared to the Natural Example",
+            combine_adjacent=True,
+        )
 
-    gr.Markdown(
-        "## <p align='center'>The Output of Reactive Perturbation Defocusing</p>"
-    )
-    with gr.Group():
-        output_is_adv_df = gr.DataFrame(label="Adversarial Example Detection Result")
         gr.Markdown(
-            "The is_adversarial field indicates an adversarial example is detected. "
-            "The perturbed_label is the predicted label of the adversarial example. "
-            "The confidence field represents the confidence of the predicted adversarial example detection. "
+            "## <h2 align='center'>The Output of Reactive Perturbation Defocusing</p>"
         )
-        output_df = gr.DataFrame(label="Repaired Standard Classification Result")
-        gr.Markdown(
-            "If is_repaired=true, it has been repaired by RPD. "
-            "The pred_label field indicates the standard classification result. "
-            "The confidence field represents the confidence of the predicted label. "
-            "The is_correct field indicates whether the predicted label is correct."
+        with gr.Row():
+            with gr.Column():
+                with gr.Group():
+                    output_is_adv_df = gr.DataFrame(
+                        label="Adversarial Example Detection Result"
+                    )
+                    gr.Markdown(
+                        "The is_adversarial field indicates if an adversarial example is detected. "
+                        "The perturbed_label is the predicted label of the adversarial example. "
+                        "The confidence field represents the confidence of the predicted adversarial example detection. "
+                    )
+            with gr.Column():
+                with gr.Group():
+                    output_df = gr.DataFrame(
+                        label="Repaired Standard Classification Result"
+                    )
+                    gr.Markdown(
+                        "If is_repaired=true, it has been repaired by RPD. "
+                        "The pred_label field indicates the standard classification result. "
+                        "The confidence field represents the confidence of the predicted label. "
+                        "The is_correct field indicates whether the predicted label is correct."
+                    )
+
+        # Bind functions to buttons
+        button_gen.click(
+            fn=run_demo,
+            inputs=[input_dataset, input_attacker, input_sentence, input_label],
+            outputs=[
+                output_original_example,
+                output_original_label,
+                output_repaired_example,
+                output_repaired_label,
+                output_adv_example,
+                ori_text_diff,
+                adv_text_diff,
+                restored_text_diff,
+                output_adv_label,
+                output_df,
+                output_is_adv_df,
+            ],
         )
 
-    gr.Markdown("## <p align='center'>Example Comparisons</p>")
-    ori_text_diff = gr.HighlightedText(
-        label="The Original Natural Example",
-        combine_adjacent=True,
-    )
-    adv_text_diff = gr.HighlightedText(
-        label="Character Editions of Adversarial Example Compared to the Natural Example",
-        combine_adjacent=True,
-    )
-    restored_text_diff = gr.HighlightedText(
-        label="Character Editions of Repaired Adversarial Example Compared to the Natural Example",
-        combine_adjacent=True,
-    )
-
-    # Bind functions to buttons
-    button_gen.click(
-        fn=generate_adversarial_example,
-        inputs=[input_dataset, input_attacker, input_sentence, input_label],
-        outputs=[
-            output_original_example,
-            output_original_label,
-            output_repaired_example,
-            output_repaired_label,
-            output_adv_example,
-            ori_text_diff,
-            adv_text_diff,
-            restored_text_diff,
-            output_adv_label,
-            output_df,
-            output_is_adv_df,
-        ],
-    )
+    demo.queue(2).launch()
 
-demo.launch()

requirements.txt

@@ -20,4 +20,5 @@ textattack[dev]
 jieba
 pycld2
 OpenHowNet
-pinyin
+pinyin
+flask

utils.py

@@ -0,0 +1,234 @@
+import random
+from difflib import Differ
+
+from textattack.attack_recipes import BAEGarg2019
+from textattack.datasets import Dataset
+from textattack.models.wrappers import HuggingFaceModelWrapper
+from findfile import find_files
+from flask import Flask
+from textattack import Attacker
+
+
+class ModelWrapper(HuggingFaceModelWrapper):
+    def __init__(self, model):
+        self.model = model  # pipeline = pipeline
+
+    def __call__(self, text_inputs, **kwargs):
+        outputs = []
+        for text_input in text_inputs:
+            raw_outputs = self.model.infer(text_input, print_result=False, **kwargs)
+            outputs.append(raw_outputs["probs"])
+        return outputs
+
+
+class SentAttacker:
+    def __init__(self, model, recipe_class=BAEGarg2019):
+        model = model
+        model_wrapper = ModelWrapper(model)
+
+        recipe = recipe_class.build(model_wrapper)
+        # WordNet defaults to english. Set the default language to French ('fra')
+
+        # recipe.transformation.language = "en"
+
+        _dataset = [("", 0)]
+        _dataset = Dataset(_dataset)
+
+        self.attacker = Attacker(recipe, _dataset)
+
+
+def diff_texts(text1, text2):
+    d = Differ()
+    return [
+        (token[2:], token[0] if token[0] != " " else None)
+        for token in d.compare(text1, text2)
+    ]
+
+
+def get_ensembled_tad_results(results):
+    target_dict = {}
+    for r in results:
+        target_dict[r["label"]] = (
+            target_dict.get(r["label"]) + 1 if r["label"] in target_dict else 1
+        )
+
+    return dict(zip(target_dict.values(), target_dict.keys()))[
+        max(target_dict.values())
+    ]
+
+
+
+def get_sst2_example():
+    filter_key_words = [
+        ".py",
+        ".md",
+        "readme",
+        "log",
+        "result",
+        "zip",
+        ".state_dict",
+        ".model",
+        ".png",
+        "acc_",
+        "f1_",
+        ".origin",
+        ".adv",
+        ".csv",
+    ]
+
+    dataset_file = {"train": [], "test": [], "valid": []}
+    dataset = "sst2"
+    search_path = "./"
+    task = "text_defense"
+    dataset_file["test"] += find_files(
+        search_path,
+        [dataset, "test", task],
+        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
+                    + filter_key_words,
+    )
+
+    for dat_type in ["test"]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+            with open(data_file, mode="r", encoding="utf8") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split("$LABEL$")
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+
+
+def get_agnews_example():
+    filter_key_words = [
+        ".py",
+        ".md",
+        "readme",
+        "log",
+        "result",
+        "zip",
+        ".state_dict",
+        ".model",
+        ".png",
+        "acc_",
+        "f1_",
+        ".origin",
+        ".adv",
+        ".csv",
+    ]
+
+    dataset_file = {"train": [], "test": [], "valid": []}
+    dataset = "agnews"
+    search_path = "./"
+    task = "text_defense"
+    dataset_file["test"] += find_files(
+        search_path,
+        [dataset, "test", task],
+        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
+                    + filter_key_words,
+    )
+    for dat_type in ["test"]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+            with open(data_file, mode="r", encoding="utf8") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split("$LABEL$")
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+
+
+def get_amazon_example():
+    filter_key_words = [
+        ".py",
+        ".md",
+        "readme",
+        "log",
+        "result",
+        "zip",
+        ".state_dict",
+        ".model",
+        ".png",
+        "acc_",
+        "f1_",
+        ".origin",
+        ".adv",
+        ".csv",
+    ]
+
+    dataset_file = {"train": [], "test": [], "valid": []}
+    dataset = "amazon"
+    search_path = "./"
+    task = "text_defense"
+    dataset_file["test"] += find_files(
+        search_path,
+        [dataset, "test", task],
+        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
+                    + filter_key_words,
+    )
+
+    for dat_type in ["test"]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+            with open(data_file, mode="r", encoding="utf8") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split("$LABEL$")
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+
+
+def get_imdb_example():
+    filter_key_words = [
+        ".py",
+        ".md",
+        "readme",
+        "log",
+        "result",
+        "zip",
+        ".state_dict",
+        ".model",
+        ".png",
+        "acc_",
+        "f1_",
+        ".origin",
+        ".adv",
+        ".csv",
+    ]
+
+    dataset_file = {"train": [], "test": [], "valid": []}
+    dataset = "imdb"
+    search_path = "./"
+    task = "text_defense"
+    dataset_file["test"] += find_files(
+        search_path,
+        [dataset, "test", task],
+        exclude_key=[".adv", ".org", ".defense", ".inference", "train."]
+                    + filter_key_words,
+    )
+
+    for dat_type in ["test"]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+            with open(data_file, mode="r", encoding="utf8") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split("$LABEL$")
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+