Merge branch 'main' into hf

README.md

@@ -42,30 +42,33 @@ QUESTION_LANG=cn QUESTION_LLM='chatgpt' QUESTION_LLM_KEY=<your API key> python3
 ```shell
 QUESTION_LANG=en QUESTION_LLM='chatgpt' QUESTION_LLM_KEY=<your API key> python3 -u app.py
 ```
-### LLaMA2-7b + 中文
+### Mistral-7B-Instruct-v0.1 + 英文
 ```shell
-QUESTION_LANG=cn QUESTION_LLM='llama2-7b' python3 -u app.py
-```
-### LLaMA2-7b + 英文
-```shell
-QUESTION_LANG=en QUESTION_LLM='llama2-7b' python3 -u app.py
+QUESTION_LANG=en QUESTION_LLM='mistral-7b' python3 -u app.py
 ```
 ## :technologist: 为什么制作这个游戏
 
 我们的目标是通过这一游戏，让参与者深入领略到提示工程（prompt engineering）和自然语言处理的令人着迷之处。这个过程将向玩家们展示，如何巧妙地构建提示词（prompts），以及如何运用它们来引发人工智能系统的惊人反应，同时也帮助他们更好地理解深度学习和自然语言处理技术的不可思议之处。
 
 ## :raising_hand: 如何提交设计好的关卡
-如果有好玩的问题或想法，欢迎玩家提交自己的创意，可以通过
+如果有好玩的问题或想法，欢迎玩家提交自己的创意，可以
 [发起 Pull Request](https://github.com/opendilab/LLMRiddles/compare) 向我们提交， 我们会在审核通过后收录至关卡中。
+问题的设计格式应包含以下几点：
+- Pull Request标题，示例：feature(username): 章节X-关卡设计
+- 希望被提及的ID
+- 对应章节问题文件的修改
+- init.py的修改
+
+完整示例请参考：[提交属于自己的关卡设计]()
 
 ## :writing_hand: 未来计划
 
 - [x] 支持自定义关卡
 - [ ] 在线试玩链接
 - [ ] Hugging Face Space 链接
-- [ ] 支持LLaMA2-7B（英文）
-- [ ] 支持Mistral-7B（英文）
+- [x] 支持Mistral-7B-Instruct-v0.1（英文）
 - [ ] 支持Baichuan2-7B（中文）
+- [ ] 支持LLaMA2-7B（英文）
 - [ ] LLM 推理速度优化
 
 

app.py

@@ -1,5 +1,6 @@
 import os
 import uuid
+import logging
 
 import gradio as gr
 
@@ -7,14 +8,20 @@ from llmriddles.questions import QuestionExecutor
 from llmriddles.questions import list_ordered_questions
 
 _QUESTION_IDS = {}
+count = 0
 _QUESTIONS = list_ordered_questions()
 _LANG = os.environ.get('QUESTION_LANG', 'cn')
 assert _LANG in ['cn', 'en'], _LANG
 _LLM = os.environ.get('QUESTION_LLM', 'chatgpt')
-assert _LLM in ['chatgpt', 'llama2-7b'], _LLM
+assert _LLM in ['chatgpt', 'mistral-7b'], _LLM
 _LLM_KEY = os.environ.get('QUESTION_LLM_KEY', None)
+_DEBUG = os.environ.get('DEBUG', 'false').lower() == 'true'
 
 if _LANG == "cn":
+    if _DEBUG:
+        logging.getLogger().setLevel(logging.INFO)
+    else:
+        logging.getLogger().setLevel(logging.WARNING)
     title = "完蛋！我被 LLM 拿捏了"
     requirement_ph = """
     欢迎来到 LLM Riddles!
@@ -122,7 +129,7 @@ if __name__ == '__main__':
                 gr_question = gr.TextArea(placeholder=question_ph, label=question_label)
                 gr_api_key = gr.Text(placeholder=api_ph, label=api_label, type='password', visible=_need_api_key())
                 with gr.Row():
-                    gr_submit = gr.Button(submit_label, interactive=True)
+                    gr_submit = gr.Button(submit_label, interactive=False)
                     gr_next = gr.Button(next_label)
 
             with gr.Column():
@@ -134,8 +141,11 @@ if __name__ == '__main__':
 
 
         def _next_question(uuid_):
+            global count
             if not uuid_:
                 uuid_ = str(uuid.uuid4())
+                count += 1
+                logging.info(f'Player {count} starts the game now')
             global _QUESTION_IDS
             _qid = _QUESTION_IDS.get(uuid_, -1)
             _qid += 1
@@ -143,8 +153,9 @@ if __name__ == '__main__':
 
             if _qid >= len(_QUESTIONS):
                 del _QUESTION_IDS[uuid_]
+                logging.info(f'Player {count} has passed the game now')
                 return game_cleared_label, '', '', {}, '', \
-                    gr.Button(submit_label, interactive=True), \
+                    gr.Button(submit_label, interactive=False), \
                     gr.Button(try_again_label, interactive=True), \
                     ''
             else:
@@ -187,4 +198,5 @@ if __name__ == '__main__':
         )
 
     concurrency = int(os.environ.get('CONCURRENCY', os.cpu_count()))
-    demo.queue().launch(max_threads=concurrency)
+    favicon_path = os.path.join(os.path.dirname(__file__), 'llmriddles', 'assets', 'avatar.png')
+    demo.queue().launch(max_threads=concurrency, favicon_path=favicon_path, share=True)

llmriddles/llms/__init__.py

@@ -1,2 +1,3 @@
-from .chatgpt import ask_chatgpt
 from .base import register_llm, get_llm_fn
+from .chatgpt import ask_chatgpt
+from .mistral import ask_mistral_7b_instruct

llmriddles/llms/llm_client.py

@@ -0,0 +1,54 @@
+import time
+import requests
+import logging
+import argparse
+
+
+class LLMFlaskClient:
+    def __init__(self, ip: str, port: int, max_retry: int = 3):
+        self.ip = ip
+        self.port = port
+
+        self.url_prefix_format = 'http://{}:{}/'
+        self.url = self.url_prefix_format.format(self.ip, self.port)
+        self.max_retry = max_retry
+
+        self.logger = logging.getLogger()
+        self.logger.addHandler(logging.StreamHandler())
+        self.logger.handlers[0].setFormatter(logging.Formatter("%(message)s"))
+
+    def _request(self, name: str, data: dict):
+        for _ in range(self.max_retry):
+            try:
+                self.logger.info(f'{name}\ndata: {data}')
+                response = requests.post(self.url + name, json=data).json()
+            except Exception as e:
+                self.logger.warning('error: ', repr(e))
+                time.sleep(1)
+                continue
+            if response['code'] == 0:
+                return response['output']
+            else:
+                raise Exception(response['error_msg'])
+        raise Exception("Web service failed. Please retry or contact with manager")
+
+    def run(self, message: str) -> str:
+        try:
+            return self._request('ask_llm_for_answer', {'user_text': message})
+        except Exception as e:
+            return f"Error: {repr(e)}"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--ip', required=True)
+    parser.add_argument('-p', '--port', required=True)
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+    if args.debug:
+        logging.getLogger().setLevel(logging.INFO)
+    else:
+        logging.getLogger().setLevel(logging.WARNING)
+
+    client = LLMFlaskClient(args.ip, args.port)
+    print(client.run('Please concatenate string "1+" and "1=3". Only give me the result without "".'))

llmriddles/llms/llm_server.py

@@ -0,0 +1,72 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from flask import Flask, request
+import argparse
+import logging
+
+
+class LLMInstance:
+
+    def __init__(self, model_path: str, device: str = "cuda"):
+
+        self.model = AutoModelForCausalLM.from_pretrained(model_path)
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path)
+        self.model.to(device)
+        self.device = device
+
+    def query(self, message):
+        try:
+            messages = [
+                {"role": "user", "content": message},
+            ]
+            encodeds = self.tokenizer.apply_chat_template(messages, return_tensors="pt")
+            model_inputs = encodeds.to(self.device)
+
+            generated_ids = self.model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+            decoded = self.tokenizer.batch_decode(generated_ids)
+
+            # output is the string decoded[0] after "[/INST]". There may exist "</s>", delete it.
+            output = decoded[0].split("[/INST]")[1].split("</s>")[0]
+            return {
+                'code': 0,
+                'ret': True,
+                'error_msg': None,
+                'output': output
+            }
+        except Exception as e:
+            return {
+                'code': 1,
+                'ret': False,
+                'error_msg': str(e),
+                'output': None
+            }
+
+
+def create_app(core):
+    app = Flask(__name__)
+
+    @app.route('/ask_llm_for_answer', methods=['POST'])
+    def ask_llm_for_answer():
+        user_text = request.json['user_text']
+        return core.query(user_text)
+
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-m', '--model_path', required=True, default='Mistral-7B-Instruct-v0.1', help='the model path of reward model')
+    parser.add_argument('--ip', default='0.0.0.0')
+    parser.add_argument('-p', '--port', default=8001)
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+    else:
+        logging.getLogger().setLevel(logging.INFO)
+    logging.getLogger().addHandler(logging.StreamHandler())
+    logging.getLogger().handlers[0].setFormatter(logging.Formatter("%(message)s"))
+
+    core = LLMInstance(args.model_path)
+    app = create_app(core)
+    app.run(host=args.ip, port=args.port)

llmriddles/llms/mistral.py

@@ -0,0 +1,22 @@
+from functools import lru_cache
+
+from .base import register_llm
+from .llm_client import LLMFlaskClient
+
+
+@lru_cache()
+def _get_mistral_7b_instruct_server(host: str, port: int):
+    from .llm_server import LLMInstance, create_app
+    core = LLMInstance('Mistral-7B-Instruct-v0.1')
+    app = create_app(core)
+    app.run(host=host, port=port)
+
+
+def ask_mistral_7b_instruct(message: str, **kwargs):
+    host, port = '0.0.0.0', 8001
+    _get_mistral_7b_instruct_server(host, port)
+    client = LLMFlaskClient(host, port)
+    return client.run(message).strip()
+
+
+register_llm('mistral-7b', ask_mistral_7b_instruct)

llmriddles/questions/__init__.py

@@ -1,7 +1,13 @@
 from .executor import QuestionExecutor
 from .level1 import __file__ as _level1_file_
+from .level2 import __file__ as _level2_file_
 from .level3 import __file__ as _level3_file_
+from .level4 import __file__ as _level4_file_
+from .level5 import __file__ as _level5_file_
 from .question import Question, register_question, list_ordered_questions
 
 _ = _level1_file_
+_ = _level2_file_
 _ = _level3_file_
+_ = _level4_file_
+_ = _level5_file_

llmriddles/questions/level2.py

@@ -0,0 +1,299 @@
+import re
+from typing import Optional, Tuple
+
+import sympy
+
+from .question import register_question
+
+CN_TEXT_1 = """
+你需要提出一个字数是质数的问题，使回答的长度刚好是它的下一个质数。
+"""
+EN_TEXT_1 = """
+You need to come up with a question that has a prime number of words, so the answer's length is exactly the next prime number.
+"""
+
+
+def _is_prime(v):
+    return sympy.isprime(v)
+
+
+def _next_prime(v):
+    while v:
+        v += 1
+        if _is_prime(v):
+            return v
+
+
+def _cn_checker_1(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    qs_length = len(user_text.strip())
+    if not _is_prime(qs_length):
+        return False, f'问题长度为{qs_length}，非质数'
+
+    answer_value = len(answer_text)
+    next_prime = _next_prime(qs_length)
+    if answer_value != next_prime:
+        return False, f'下一个质数为{next_prime}，但回答长度为{answer_value}'
+
+    return True, None
+
+
+def _en_words(text: str):
+    return len(re.findall(r'\w+', text))
+
+
+def _en_checker_1(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    qs_length = _en_words(user_text.strip())
+    if not _is_prime(qs_length):
+        return False, f'The question has a length of {qs_length}, which is not a prime number'
+
+    answer_value = _en_words(answer_text)
+    next_prime = _next_prime(qs_length)
+    if answer_value != next_prime:
+        return False, f'The next prime number is {next_prime}, but the answer\'s length is {answer_value}'
+
+    return True, None
+
+
+register_question({
+    'cn': CN_TEXT_1,
+    'en': EN_TEXT_1,
+}, checkers={
+    'cn': _cn_checker_1,
+    'en': _en_checker_1,
+})
+
+CN_TEXT_2 = """
+请输入一个大于一的正整数作为问题，使回答里包含至少比它大一千的数。
+"""
+EN_TEXT_2 = """
+Please enter a positive integer greater than one as a question, so that the answer contains a number that is at least one thousand more than it.
+"""
+
+
+def _cn_checker_2(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'输入内容{user_text!r}，并非一个大于1的正整数'
+
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item >= value + 1000:
+            return True, f'检测到输出中数字{value_item}，满足要求'
+
+    return False, f'未在输出中检测到不少于{value + 1000}的数字'
+
+
+def _en_checker_2(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'You entered {user_text!r}, which is not a positive integer greater than 1'
+
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item >= value + 1000:
+            return True, f'Detected the number {value_item} in the output, which meets the requirement'
+
+    return False, f'Did not detect a number of at least {value + 1000} in the output'
+
+
+register_question({
+    'cn': CN_TEXT_2,
+    'en': EN_TEXT_2,
+}, checkers={
+    'cn': _cn_checker_2,
+    'en': _en_checker_2,
+})
+
+CN_TEXT_3 = """
+请输入一个大于一的正整数作为问题，使回答里包含至少10个不同的至少比它小一千的自然数。
+"""
+EN_TEXT_3 = """
+Please enter a positive integer greater than one as a question, so that the answer contains at least 10 different natural numbers, each of them at least 1000 less than the question.
+"""
+
+
+def _cn_checker_3(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'输入内容{user_text!r}，并非一个大于1的正整数'
+
+    collected_values = []
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item <= value - 1000:
+            collected_values.append(value_item)
+
+    collected_values = sorted(set(collected_values))
+    if len(collected_values) >= 10:
+        return True, f'检测到{len(collected_values)}个不大于{value - 1000}的数字：{collected_values!r}'
+    else:
+        return False, f'检测到{len(collected_values)}个不大于{value - 1000}的数字,未达到10个：{collected_values!r}'
+
+
+def _en_checker_3(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'You entered {user_text!r}, which is not a positive integer greater than 1'
+
+    collected_values = []
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item <= value - 1000:
+            collected_values.append(value_item)
+
+    collected_values = sorted(set(collected_values))
+    if len(collected_values) >= 10:
+        return True, f'Detected {len(collected_values)} numbers not greater than {value - 1000}: {collected_values!r}'
+    else:
+        return False, f'Detected {len(collected_values)} numbers not greater than {value - 1000}, ' \
+                      f'but it doesn\'t reach 10: {collected_values!r}'
+
+
+register_question({
+    'cn': CN_TEXT_3,
+    'en': EN_TEXT_3,
+}, checkers={
+    'cn': _cn_checker_3,
+    'en': _en_checker_3,
+})
+
+CN_TEXT_4 = """
+请在不提及1,4,5这三个字符的情况下让模型回答114514，不要有多余的字符。
+"""
+EN_TEXT_4 = """
+Please make the model answer "114514" without mentioning the characters 1, 4, and 5, and without any extra characters.
+"""
+
+
+def _cn_checker_4(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    if "1" in user_text or "4" in user_text or "5" in user_text:
+        return False, '请不要在提问中出现1、4、5三个数字'
+
+    return answer_text.strip() == '114514', None
+
+
+def _en_checker_4(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    if "1" in user_text or "4" in user_text or "5" in user_text:
+        return False, 'Please don\'t include the numbers 1, 4, and 5 in the question'
+
+    return answer_text.strip() == '114514', None
+
+
+register_question({
+    'cn': CN_TEXT_4,
+    'en': EN_TEXT_4,
+}, checkers={
+    'cn': _cn_checker_4,
+    'en': _en_checker_4,
+})
+
+CN_TEXT_5 = """
+请输入一个大于一的平方数作为问题，使回答里包含它的平方根对应的数。
+"""
+EN_TEXT_5 = """
+Please provide a square number greater than one as the question, and the answer should include the number corresponding to its square root.
+"""
+
+
+def _cn_checker_5(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+        if int(value ** 0.5) ** 2 != value:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'输入内容{user_text!r}，并非一个大于1的平方数'
+
+    sq = int(value ** 0.5)
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item == sq:
+            return True, f'检测到平方根数{sq}'
+    else:
+        return False, f'未检测到平方根数{sq}'
+
+
+def _en_checker_5(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+        if int(value ** 0.5) ** 2 != value:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'The input, {user_text!r}, is not a square number greater than 1'
+
+    sq = int(value ** 0.5)
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item == sq:
+            return True, f'Detected square root: {sq}'
+    else:
+        return False, f'Square root {sq} not detected'
+
+
+register_question({
+    'cn': CN_TEXT_5,
+    'en': EN_TEXT_5,
+}, checkers={
+    'cn': _cn_checker_5,
+    'en': _en_checker_5,
+})
+
+CN_TEXT_6 = """
+请提一个不包含“狗”这个字的10个字以内的问题，使回答中“狗”这个字出现的次数至少是问题字数的2倍。
+"""
+EN_TEXT_6 = """
+Please ask a question in under 10 words without the word "dog" and ensure that the word "dog" appears at least twice in the answer for each word in the question.
+"""
+
+
+def _cn_checker_6(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    if len(user_text) > 10:
+        return False, '问题不得超过10个字'
+    if '狗' in user_text:
+        return False, '问题不得包含“狗”字'
+
+    dog_count = len(re.findall('狗', answer_text))
+    if dog_count >= len(user_text) * 2:
+        return True, f'“狗”字的出现次数为{dog_count}次'
+    else:
+        return False, f'“狗”字的出现次数为{dog_count}次，未达到{len(user_text) * 2}次'
+
+
+def _en_checker_6(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    q_words = re.findall(r'\w+', user_text.lower())
+    if len(q_words) > 10:
+        return False, 'The question must not exceed 10 words'
+    if any(word in {'dog', 'dogs'} for word in q_words):
+        return False, 'The question must not contain the word "dog" or "dogs"'
+
+    a_words = re.findall(r'\w+', answer_text.lower())
+    a_dog_count = sum(1 if word in {'dog', 'dogs'} else 0 for word in a_words)
+    if a_dog_count >= len(q_words) * 2:
+        return True, f'The word "dog" (or "dogs") appears {a_dog_count} times.'
+    else:
+        return False, f'The word "dog" (or "dogs") appears {a_dog_count} times, ' \
+                      f'which is less than {len(q_words) * 2} times.'
+
+
+register_question({
+    'cn': CN_TEXT_6,
+    'en': EN_TEXT_6,
+}, checkers={
+    'cn': _cn_checker_6,
+    'en': _en_checker_6,
+})

llmriddles/questions/level4.py

@@ -0,0 +1,97 @@
+from .question import register_question
+import re
+
+
+def check_if_is_number(text: str):
+    try:
+        int(text)
+        return True
+    except ValueError:
+        return False
+
+
+def get_all_numbers_in_a_sentence(text: str):
+    return [int(i) for i in re.findall(r'\d+', text)]
+
+
+# CN_TEXT_1 = """
+# 第四章第一题，请构造一个问题A，它的回复是不同于A的B，然后以B提问能再次得到A。
+
+# 请在下面的输入框内填写你构造并点击按钮提交。
+# """
+
+# EN_TEXT_1 = """
+# For the first question in chapter 4, please make a question A whose answer is B that is different from A, and then ask B to get A again.
+
+# Please enter your query below and click the submit button
+# """
+
+
+# def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+#     user_text = user_text.strip()
+#     pass
+
+# register_question({
+#     'cn': CN_TEXT_1,
+#     'en': EN_TEXT_1,
+# }, _checker_1, level=4)
+
+
+# CN_TEXT_2 = """
+# 第四章第二题，
+
+# 请在下面的输入框内填写你构造并点击按钮提交。
+# """
+
+# EN_TEXT_2 = """
+# For the second question in chapter 4,
+
+# Please enter your query below and click the submit button
+# """
+
+
+# def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+#     user_text = user_text.strip()
+#     pass
+
+# register_question({
+#     'cn': CN_TEXT_2,
+#     'en': EN_TEXT_2,
+# }, _checker_2, level=4)
+
+
+CN_TEXT_3 = """
+第四章第三题（自然之密），请输入一个大于一的正整数作为问题，使回答里包含和它刚好相差1的数。
+
+请在下面的输入框内填写你构造并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the third question in chapter 4, please enter a positive integer greater than one as the question so that the answer contains a number that is exactly 1 different from it.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+    if not check_if_is_number(question_text):
+        return False, "问题应该是一个正整数" if lang == 'cn' else 'Question should be a positive integer.'
+    elif int(question_text) == 1:
+        return False, "问题应该是一个大于1的正整数" if lang == 'cn' else 'Question should be a positive integer greater than 1.'
+    elif int(question_text)-1 not in get_all_numbers_in_a_sentence(answer_text) or int(question_text)+1 not in get_all_numbers_in_a_sentence(answer_text):
+        return False, "回答中应该包含一个与问题相差1的数字" if lang == 'cn' else 'Answer should contain a number that is exactly 1 different from the question.'
+    else:
+        return True, None
+
+
+register_question({
+    'cn': CN_TEXT_3,
+    'en': EN_TEXT_3,
+}, _checker_3, level=4)

llmriddles/questions/level5.py

@@ -0,0 +1,56 @@
+from .question import register_question
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words)+len(english_words)+len(other_words)
+    else:
+        return len(chinese_words)+len(english_words)
+
+CN_TEXT_1 = """
+第五章第一题（口是心非）请构造一个不少于十个字的问题，使得回答中不包含问题中的任意字符。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 5, Please construct a question of no less than ten words so that the answer does not contain any of the characters in the question.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) < 10:
+        return False, "用户的问题长度应该至少10个字" if lang=='cn' else 'Question should be no less than 10 words.'
+
+    if answer_text == user_text:
+        return True, None
+    else:
+        return False, None
+
+register_question({
+    'cn': CN_TEXT_1,
+    'en': EN_TEXT_1,
+}, _checker_1, level=5)
+

llmriddles/questions/question.py

@@ -23,8 +23,8 @@ def register_question(text: Union[Mapping[str, str], str],
     if isinstance(checkers, collections.abc.Mapping):
         _origin_checkers = checkers
 
-        def _integrated_checker(question_text: str, answer_text: str, lang: str):
-            return _origin_checkers[lang](question_text, answer_text)
+        def _integrated_checker(question_text: str, user_text: str, answer_text: str, lang: str):
+            return _origin_checkers[lang](question_text, user_text, answer_text)
 
         checker: MultiLangCheckerTyping = _integrated_checker
     else:

requirements.txt

@@ -2,4 +2,7 @@ hbutils>=0.9.1
 tqdm
 requests>=2.20
 gradio==4.1.1
-openai>=1
+openai>=1
+sympy
+flask
+transformers