dev(hansbug): merge from main

README.md

@@ -11,11 +11,77 @@ license: apache-2.0
 python_version: 3.8
 ---
 
-# LLMRiddles
+# LLM Riddles
 
-```shell
-# QUESTION_LANG=en
-# QUESTION_LLM=chatgpt
+<!-- <div align="center">
+	<br>
+	<a href="https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/banner.svg">
+		<img src="https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/banner.svg" width="1000" height="200" alt="Click to see the source">
+	</a>
+	<br>
+</div> -->
+
+<div align="center">
+	<br>
+	<a href="https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/banner.svg">
+		<img src="https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/banner.svg" width="1000" height="200" alt="Click to see the source">
+	</a>
+	<br>
+</div>
+
+## :thinking: 什么是LLM Riddles
+欢迎来到 LLM Riddles！这是一个与语言模型斗智斗勇的游戏。在游戏中，你需要构造与语言模型交互的问题，来得到符合要求的答案。在这个过程中，你可以开动脑筋，用你想到的所有方式，让模型输出答案要求的结果。
 
-python app.py
+## :space_invader: 如何试玩
+我们提供了在线版本以供玩家直接访问试玩，本地部署可以通过以下方式：
+### ChatGPT + 中文
+```shell
+QUESTION_LANG=cn QUESTION_LLM='chatgpt' QUESTION_LLM_KEY=<your API key> python3 -u app.py
+```
+### ChatGPT + 英文
+```shell
+QUESTION_LANG=en QUESTION_LLM='chatgpt' QUESTION_LLM_KEY=<your API key> python3 -u app.py
+```
+### LLaMA2-7b + 中文
+```shell
+QUESTION_LANG=cn QUESTION_LLM='llama2-7b' python3 -u app.py
+```
+### LLaMA2-7b + 英文
+```shell
+QUESTION_LANG=en QUESTION_LLM='llama2-7b' python3 -u app.py
 ```
+## :technologist: 为什么制作这个游戏
+
+我们的目标是通过这一游戏，让参与者深入领略到提示工程（prompt engineering）和自然语言处理的令人着迷之处。这个过程将向玩家们展示，如何巧妙地构建提示词（prompts），以及如何运用它们来引发人工智能系统的惊人反应，同时也帮助他们更好地理解深度学习和自然语言处理技术的不可思议之处。
+
+## :raising_hand: 如何提交设计好的关卡
+如果有好玩的问题或想法，欢迎玩家提交自己的创意，可以通过
+[发起 Pull Request](https://github.com/opendilab/LLMRiddles/compare) 向我们提交， 我们会在审核通过后收录至关卡中。
+
+## :writing_hand: 未来计划
+
+- [x] 支持自定义关卡
+- [ ] 在线试玩链接
+- [ ] Hugging Face Space 链接
+- [ ] 支持LLaMA2-7B（英文）
+- [ ] 支持Mistral-7B（英文）
+- [ ] 支持Baichuan2-7B（中文）
+- [ ] LLM 推理速度优化
+
+
+## :speech_balloon: 反馈问题 & 提出建议
+- 在 GitHub 上[发起 Issue](https://github.com/opendilab/CodeMorpheus/issues/new/choose)
+- 通过邮件与我们联系 ([email protected])
+
+- 在OpenDILab的群组中加入讨论(通过 WeChat: ding314assist 添加小助手微信)
+<img src=https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/wechat.jpeg width=35% />
+
+## Special Thanks
+- 感谢 [Haoqiang Fan](https://www.zhihu.com/people/haoqiang-fan) 的原始创意和题目，为本项目的开发和扩展提供了灵感与动力。
+- 感谢 [HuggingFace](https://huggingface.co) 对游戏的支持与协助。
+- 感谢 [LLM Riddles contributors](https://github.com/opendilab/LLMRiddles/graphs/contributors) 的实现与支持。
+
+## License
+All code within this repository is under [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0).
+
+<p align="right">(<a href="#top">back to top</a>)</p>

app.py

@@ -9,11 +9,99 @@ from llmriddles.questions import list_ordered_questions
 _QUESTION_IDS = {}
 _QUESTIONS = list_ordered_questions()
 _LANG = os.environ.get('QUESTION_LANG', 'cn')
+assert _LANG in ['cn', 'en'], _LANG
 _LLM = os.environ.get('QUESTION_LLM', 'chatgpt')
+assert _LLM in ['chatgpt', 'llama2-7b'], _LLM
+_LLM_KEY = os.environ.get('QUESTION_LLM_KEY', None)
+
+if _LANG == "cn":
+    title = "完蛋！我被 LLM 拿捏了"
+    requirement_ph = """
+    欢迎来到 LLM Riddles!
+
+    你将通过本游戏对大语言模型产生更深刻的理解。在本游戏中，你需要构造一个提给语言大模型的问题，使得它回复的答案符合题目要求。
+
+    点击\"下一题\"即可开始游戏
+    """
+    requirement_label = "游戏须知/说明"
+    question_ph = "你对大语言模型的提问"
+    question_label = "玩家提问栏"
+    answer_ph = "大语言模型的回答"
+    answer_label = "大语言模型回答栏"
+    submit_label = "提交"
+    next_label = "下一题"
+    api_ph = "你个人的大语言模型 API Key (例如：ChatGPT)"
+    api_label = "API key"
+    predict_label = "结果正确性"
+    explanation_label = "结果详细解释"
+    game_cleared_label = "祝贺！你已成功通关！"
+    correct_label = "正确"
+    wrong_label = "错误"
+    api_error_info = "请在提交问题之前先输入你的 API Key"
+    try_again_label = "再玩一次"
+    title_markdown = """
+    <div align="center">
+        <img src="https://raw.githubusercontent.com/opendilab/LLMRiddles/main/llmriddles/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
+    </div>
+    <h2 style="text-align: center; color: black;"><a href="https://github.com/OpenDILab"> 🎭LLM Riddles：完蛋！我被 LLM 拿捏了</a></h2>
+    <h4 align="center"> 如果你喜欢这个项目，请给我们在 GitHub 点个 star ✨ 。我们将会持续保持更新。再次感谢游戏<a href="https://www.zhihu.com/people/haoqiang-fan"> 原作者 </a>的奇思妙想！  </h4>
+    <strong><h5 align="center">注意：算法模型的输出可能包含一定的随机性。相关结果不代表任何开发者和相关 AI 服务的态度和意见。本项目开发者不对生成结果作任何保证，仅供娱乐。<h5></strong>
+    """
+    tos_markdown = """
+    ### 使用条款
+    玩家使用本服务须同意以下条款：
+    该服务是一项探索性研究预览版，仅供非商业用途。它仅提供有限的安全措施，并可能生成令人反感的内容。不得将其用于任何非法、有害、暴力、种族主义等目的。该服务可能会收集玩家对话数据以供未来研究之用。
+    如果您的游玩体验有不佳之处，请发送邮件至 [email protected] ！ 我们将删除相关信息，并不断改进这个项目。
+    为了获得最佳体验，请使用台式电脑进行此预览版游戏，因为移动设备可能会影响可视化效果。
+    **版权所有 2023 OpenDILab。**
+    """
+elif _LANG == "en":
+    title = "LLM Riddles: Oops! Rolling in LLM."
+    requirement_ph = """
+    Welcome to LLM Riddles!
+
+    In this game, you'll gain a deeper understanding of language models. Your challenge is to create a question to ask a language model in a way that the answer it provides meets specific criteria.
+
+    Click \'Next\' to Start
+    """
+    requirement_label = "Game Requirements"
+    question_ph = "Your Question for LLM"
+    question_label = "Question"
+    answer_ph = "Answer From LLM"
+    answer_label = "Answer"
+    submit_label = "Submit"
+    next_label = "Next"
+    api_ph = "Your API Key (e.g. ChatGPT)"
+    api_label = "API key"
+    predict_label = "Correctness"
+    explanation_label = "Explanation"
+    game_cleared_label = "Congratulations!"
+    correct_label = "Correct"
+    wrong_label = "Wrong"
+    api_error_info = "Please Enter API Key Before Submitting Question."
+    try_again_label = "Try Again"
+    title_markdown = """
+    <div align="center">
+        <img src="https://raw.githubusercontent.com/opendilab/LLMRiddles/main/llmriddles/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
+    </div>
+    <h2 style="text-align: center; color: black;"><a href="https://github.com/OpenDILab"> 🎭LLM Riddles: Oops! Rolling in LLM.</a></h2>
+    <h4 align="center"> If you like our project, please give us a star ✨ on GitHub for latest update. Thanks for the interesting idea of the original game <a href="https://www.zhihu.com/people/haoqiang-fan"> author </a>.  </h4>
+    <strong><h5 align="center">Notice: The output is generated by algorithm scheme and may involve some randomness. It does not represent the attitudes and opinions of any developers and AI services in this project. We do not make any guarantees about the generated content.<h5></strong>
+    """
+    tos_markdown = """
+    ### Terms of use
+    By using this service, players are required to agree to the following terms:
+    The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
+    Please send email to [email protected] if you get any inappropriate answer! We will delete those and keep improving our moderator.
+    For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+    **Copyright 2023 OpenDILab.**
+    """
+else:
+    raise KeyError("invalid _LANG: {}".format(_LANG))
 
 
 def _need_api_key():
-    return _LLM == 'chatgpt'
+    return _LLM == 'chatgpt' and _LLM_KEY is None
 
 
 def _get_api_key_cfgs(api_key):
@@ -24,21 +112,25 @@ def _get_api_key_cfgs(api_key):
 
 
 if __name__ == '__main__':
-    with gr.Blocks() as demo:
+    with gr.Blocks(title=title, theme='ParityError/Interstellar') as demo:
+        gr.Markdown(title_markdown)
+
+        with gr.Row():
+            gr_requirement = gr.TextArea(placeholder=requirement_ph, label=requirement_label, lines=4)
         with gr.Row():
             with gr.Column():
-                gr_requirement = gr.TextArea(placeholder='Click \'Next\' to Start', label='Requirements')
-                gr_question = gr.TextArea(placeholder='Your Question for LLM', label='Question')
-                gr_answer = gr.TextArea(placeholder='Answer From LLM', label='Answer')
-                gr_submit = gr.Button('Submit', interactive=False)
+                gr_question = gr.TextArea(placeholder=question_ph, label=question_label)
+                gr_api_key = gr.Text(placeholder=api_ph, label=api_label, type='password', visible=_need_api_key())
+                with gr.Row():
+                    gr_submit = gr.Button(submit_label, interactive=True)
+                    gr_next = gr.Button(next_label)
 
             with gr.Column():
-                gr_api_key = gr.Text(placeholder='Your API Key', label='API Key', type='password',
-                                     visible=_need_api_key())
                 gr_uuid = gr.Text(value='', visible=False)
-                gr_predict = gr.Label(label='Correctness')
-                gr_explanation = gr.TextArea(label='Explanation')
-                gr_next = gr.Button('Next')
+                gr_predict = gr.Label(label=predict_label)
+                gr_answer = gr.TextArea(label=answer_label, lines=3)
+                gr_explanation = gr.TextArea(label=explanation_label, lines=1)
+        gr.Markdown(tos_markdown)
 
 
         def _next_question(uuid_):
@@ -48,22 +140,20 @@ if __name__ == '__main__':
             _qid = _QUESTION_IDS.get(uuid_, -1)
             _qid += 1
             _QUESTION_IDS[uuid_] = _qid
-            print(_QUESTION_IDS)
 
             if _qid >= len(_QUESTIONS):
                 del _QUESTION_IDS[uuid_]
-                return 'Congratulations!', '', '', {}, '', \
-                    gr.Button('Submit', interactive=False), \
-                    gr.Button('Try Again', interactive=True), \
+                return game_cleared_label, '', '', {}, '', \
+                    gr.Button(submit_label, interactive=True), \
+                    gr.Button(try_again_label, interactive=True), \
                     ''
             else:
                 executor = QuestionExecutor(_QUESTIONS[_qid], _LANG)
                 return executor.question_text, '', '', {}, '', \
-                    gr.Button('Submit', interactive=True), \
-                    gr.Button('Next', interactive=False), \
+                    gr.Button(submit_label, interactive=True), \
+                    gr.Button(next_label, interactive=False), \
                     uuid_
 
-
         gr_next.click(
             fn=_next_question,
             inputs=[gr_uuid],
@@ -76,22 +166,19 @@ if __name__ == '__main__':
 
         def _submit_answer(qs_text: str, api_key: str, uuid_: str):
             if _need_api_key() and not api_key:
-                return '---', {}, 'Please Enter API Key Before Submitting Question.', \
-                    gr.Button('Next', interactive=False), uuid_
+                raise gr.Error(api_error_info)
 
-            print(_QUESTION_IDS)
             _qid = _QUESTION_IDS[uuid_]
             executor = QuestionExecutor(
                 _QUESTIONS[_qid], _LANG,
-                llm=_LLM, llm_cfgs=_get_api_key_cfgs(api_key) if _need_api_key() else {}
+                llm=_LLM, llm_cfgs=_get_api_key_cfgs(api_key) if _need_api_key() else {'api_key': _LLM_KEY}
             )
             answer_text, correctness, explanation = executor.check(qs_text)
-            labels = {'Correct': 1.0} if correctness else {'Wrong': 1.0}
+            labels = {correct_label: 1.0} if correctness else {wrong_label: 1.0}
             if correctness:
-                return answer_text, labels, explanation, gr.Button('Next', interactive=True), uuid_
+                return answer_text, labels, explanation, gr.Button(next_label, interactive=True), uuid_
             else:
-                return answer_text, labels, explanation, gr.Button('Next', interactive=False), uuid_
-
+                return answer_text, labels, explanation, gr.Button(next_label, interactive=False), uuid_
 
         gr_submit.click(
             _submit_answer,
@@ -99,4 +186,5 @@ if __name__ == '__main__':
             outputs=[gr_answer, gr_predict, gr_explanation, gr_next, gr_uuid],
         )
 
-    demo.launch()
+    concurrency = int(os.environ.get('CONCURRENCY', os.cpu_count()))
+    demo.queue().launch(max_threads=concurrency)

llmriddles/questions/__init__.py

@@ -1,5 +1,7 @@
 from .executor import QuestionExecutor
 from .level1 import __file__ as _level1_file_
+from .level3 import __file__ as _level3_file_
 from .question import Question, register_question, list_ordered_questions
 
 _ = _level1_file_
+_ = _level3_file_

llmriddles/questions/executor.py

@@ -17,11 +17,11 @@ class QuestionExecutor:
 
     def check(self, qs_text: str) -> Tuple[str, bool, str]:
         answer_text = get_llm_fn(self.llm)(qs_text, **self.llm_cfgs)
-        correct, explanation = self.check_answer(answer_text)
+        correct, explanation = self.check_answer(qs_text, answer_text)
         return answer_text, correct, explanation
 
-    def check_answer(self, answer_text: str) -> Tuple[bool, str]:
-        correct, explanation = self.question.checker(self.question_text, answer_text, self.lang)
+    def check_answer(self, user_text: str, answer_text: str) -> Tuple[bool, str]:
+        correct, explanation = self.question.checker(self.question_text, user_text, answer_text, self.lang)
         if explanation is None:
             if correct:
                 explanation = 'LLM的回答满足要求' if self.lang == 'cn' else 'Correct Answer From LLM'

llmriddles/questions/level1.py

@@ -1,40 +1,184 @@
-from .question import register_question
-
-CN_TEXT = """
-欢迎来玩LLM Riddles!
-
-你将通过本游戏对语言大模型产生更深刻的理解。
-
-在本游戏中，你需要构造一个提给一个语言大模型的问题，使得它回复的答案符合要求。
-
-作为第一个题目，请你构造一个问题使模型的回答是一字不差的“1+1=3”（不需要引号）。
-
-请在下面的输入框内填写你构造并点击按钮提交。
-"""
-
-EN_TEXT = """
-Welcome to LLM Riddles!
-
-In this game, you'll gain a deeper understanding of language models.
-
-Your challenge is to create a question to ask a language model in a way that the answer it provides meets specific criteria.
-
-For the first question, please construct a query for the model that will result in the exact answer "1+1=3" (no quotes needed).
-
-Please enter your query below and click the submit button
-"""
-
-
-def _checker(question_text: str, answer_text: str, lang: str):
-    _ = question_text, lang
-    answer_text = answer_text.strip()
-    if answer_text == '1+1=3':
-        return True, None
-    else:
-        return False, None
-
-
-register_question({
-    'cn': CN_TEXT,
-    'en': EN_TEXT,
-}, _checker)
+from .question import register_question
+
+def count_english_words(text: str):
+    return len(text.split(' '))
+
+def count_chinese_words(text: str):
+    return len(text)
+
+def check_if_chinese(text: str):
+    return all('\u4e00' <= char <= '\u9fff' for char in text)
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words)+len(english_words)+len(other_words)
+    else:
+        return len(chinese_words)+len(english_words)
+
+
+CN_TEXT_1 = """
+第一章第一题（初来乍到），请你构造一个问题使模型的回答是一字不差的“1+1=3”（不需要引号）。
+
+请在下面的输入框内填写你的构造并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 1, please construct a query for the model that will result in the exact answer "1+1=3" (no quotes needed).
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+    if answer_text == '1+1=3':
+        return True, None
+    else:
+        return False, None
+
+
+register_question({
+    'cn': CN_TEXT_1,
+    'en': EN_TEXT_1,
+}, _checker_1)
+
+
+CN_TEXT_2 = """
+第一章第二题（小试牛刀），请你输入三个字（及）以内的问题，使模型的回答在30个字以上。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_2 = """
+For the second question in chapter 1, please enter a question within three words so that the model’s answer is more than 30 words.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 3:
+        return False, "用户的问题长度应该在三个字及以内" if lang == 'cn' else 'Question should be within three words.'
+    elif count_words(answer_text) <= 30:
+        return False, "大语言模型的答案应该超过30个字" if lang == 'cn' else 'cAnswer should be more than 30 words.'
+    else:
+        return True, None
+
+
+register_question({
+    'cn': CN_TEXT_2,
+    'en': EN_TEXT_2,
+}, _checker_2)
+
+
+CN_TEXT_3 = """
+第一章第三题（短说长话），请你输入一个字的问题，使模型的回答在100个字以上。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the third question in chapter 1, please enter a one-word question so that the model’s answer is more than 100 words.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 1:
+        return False, "用户的问题长度应该在一个字及以内" if lang == 'cn' else 'Question should be one word.'
+    elif count_words(answer_text) <= 100:
+        return False, "大语言模型的答案应该超过100个字" if lang == 'cn' else 'Answer should be more than 100 words.'
+    else:
+        return True, None
+
+register_question({
+    'cn': CN_TEXT_3,
+    'en': EN_TEXT_3,
+}, _checker_3)
+
+
+CN_TEXT_4 = """
+第一章第四题（短说短话），请输入一个字的问题，使模型的回答字数小于20个字。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_4 = """
+For the fourth question in chapter 1, please enter a one-word question so that the model’s answer is less than 20 words.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 1:
+        return False, "用户的问题长度应该在一个字及以内" if lang == 'cn' else 'Question should be one word.'
+    elif count_words(answer_text) >= 20:
+        return False, "大语言模型的答案应该小于100个字" if lang == 'cn' else 'Answer should be less than 20 words.'
+    else:
+        return True, None
+
+register_question({
+    'cn': CN_TEXT_4,
+    'en': EN_TEXT_4,
+}, _checker_4)
+
+
+# CN_TEXT_5 = """
+# 第一章第五题（回文不变），请输入一个本身不是回文串的问题，使无论正着问还是倒着问，模型的回答是一样的。
+
+# 请在下面的输入框内填写你的问题并点击按钮提交。
+# """
+
+# EN_TEXT_5 = """
+# For the fourth question in chapter 1, please enter a question that is not a palindrome string so that the model's answer is the same whether it is asked forward or backward.
+
+# Please enter your query below and click the submit button
+# """
+
+# def _checker_5(question_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+
+#     if count_words(question_text) > 0:
+#         return False, 'Question should be one word.'
+#     elif count_words(answer_text) >= 20:
+#         return False, 'Answer should be less than 20 words.'
+#     else:
+#         return True, None
+
+# register_question({
+#     'cn': CN_TEXT_5,
+#     'en': EN_TEXT_5,
+# }, _checker_5)

llmriddles/questions/level3.py

@@ -0,0 +1,142 @@
+from .question import register_question
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words)+len(english_words)+len(other_words)
+    else:
+        return len(chinese_words)+len(english_words)
+
+CN_TEXT_1 = """
+第三章第一题（自说自话），请你输入一个至少10个字的提问，使模型的回答和提问完全一样。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 3, please enter a question of at least ten words so that the model's answer is exatly the same with the question.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) < 10:
+        return False, "用户的问题长度应该至少10个字" if lang=='cn' else 'Question should be no less than 10 words.'
+
+    if answer_text == user_text:
+        return True, None
+    else:
+        return False, None
+
+register_question({
+    'cn': CN_TEXT_1,
+    'en': EN_TEXT_1,
+}, _checker_1, level=3)
+
+CN_TEXT_2 = """
+第三章第二题（话自说自），请你输入一个至少10个字的提问，使模型的回答是问题的反序。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_2 = """
+For the second question in chapter 3, please enter a question of at least ten words so that the model's answer is the reverse order of the question.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) < 10:
+        return False, "用户的问题长度应该至少10个字" if lang=='cn' else 'Question should be no less than 10 words.'
+
+    if lang=='cn':
+        user_text = user_text[::-1]
+    else:
+        words = user_text.split()
+        words.reverse()
+        user_text = ' '.join(words)
+
+    if user_text == answer_text:
+        return True, None
+    else:
+        return False, None
+
+register_question({
+    'cn': CN_TEXT_2,
+    'en': EN_TEXT_2,
+}, _checker_2, level=3)
+
+CN_TEXT_3 = """
+第三章第三题（就是敢说），请你输入一个不包含“1”和“3”的提问，使模型的回答是一字不差的“1+1=3”（不需要引号）。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the third question in chapter 3, please enter a question not contain "1" or "3" that will result in the exact answer "1+1=3" (no quotes needed).
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if '1' in user_text or '3' in user_text:
+        return False, "用户的问题不能包含“1”或“3”" if lang=='cn' else 'Question should not contain \"1\" or \"3\".'
+
+    if answer_text == '1+1=3':
+        return True, None
+    else:
+        return False, None
+
+register_question({
+    'cn': CN_TEXT_3,
+    'en': EN_TEXT_3,
+}, _checker_3, level=3)
+
+# CN_TEXT_4 = """
+# 第三章第四题（回文协变），请你输入一个本身不是回文串的问题，使得正着问和倒着问时，模型的回答本身不是回文且也是逆序。
+
+# 请在下面的输入框内填写你的提问并点击按钮提交。
+# """
+
+# EN_TEXT_4 = """
+# For the fourth question in chapter 3, please enter a question that is not a palindrome string, so that the model's answer is also not a palindrome and is in reverse order when asked forward or backward.
+
+# Please enter your query below and click the submit button
+# """
+
+# def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
+#     pass
+
+# register_question({
+#     'cn': CN_TEXT_4,
+#     'en': EN_TEXT_4,
+# }, _checker_4, level=3)