Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files- .gitattributes +36 -35
- README.md +8 -14
- app.py +419 -0
- faiss_index_hnsw_new/index.faiss +3 -0
- faiss_index_hnsw_new/index.pkl +3 -0
- requirements.txt +0 -0
.gitattributes
CHANGED
@@ -1,35 +1,36 @@
|
|
1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
faiss_index_hnsw_new/index.faiss filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
@@ -1,14 +1,8 @@
|
|
1 |
-
---
|
2 |
-
title:
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
pinned: false
|
10 |
-
license: mit
|
11 |
-
short_description: ai李敖测试服
|
12 |
-
---
|
13 |
-
|
14 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
1 |
+
---
|
2 |
+
title: aileeao_test
|
3 |
+
app_file: app.py
|
4 |
+
sdk: gradio
|
5 |
+
sdk_version: 5.20.0
|
6 |
+
---
|
7 |
+
|
8 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
|
|
|
|
|
|
|
|
app.py
ADDED
@@ -0,0 +1,419 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import gradio as gr
|
3 |
+
import requests
|
4 |
+
from langchain_community.document_loaders import TextLoader, DirectoryLoader
|
5 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
6 |
+
from langchain_community.vectorstores import FAISS
|
7 |
+
from langchain_openai import ChatOpenAI
|
8 |
+
from langchain.prompts import PromptTemplate
|
9 |
+
import numpy as np
|
10 |
+
import faiss
|
11 |
+
from collections import deque
|
12 |
+
from langchain_core.embeddings import Embeddings
|
13 |
+
import threading
|
14 |
+
import queue
|
15 |
+
from langchain_core.messages import HumanMessage, AIMessage
|
16 |
+
from sentence_transformers import SentenceTransformer
|
17 |
+
import pickle
|
18 |
+
import torch
|
19 |
+
from langchain_core.documents import Document
|
20 |
+
import time
|
21 |
+
from tqdm import tqdm
|
22 |
+
|
23 |
+
# 获取环境变量
|
24 |
+
os.environ["OPENROUTER_API_KEY"] = os.getenv("OPENROUTER_API_KEY", "")
|
25 |
+
if not os.environ["OPENROUTER_API_KEY"]:
|
26 |
+
raise ValueError("OPENROUTER_API_KEY 未设置,请在环境变量中配置或在 .env 文件中添加")
|
27 |
+
SILICONFLOW_API_KEY = os.getenv("SILICONFLOW_API_KEY")
|
28 |
+
if not SILICONFLOW_API_KEY:
|
29 |
+
raise ValueError("SILICONFLOW_API_KEY 未设置,请在 Hugging Face Spaces 的 Settings > Secrets 中添加 SILICONFLOW_API_KEY")
|
30 |
+
|
31 |
+
# SiliconFlow API 配置
|
32 |
+
SILICONFLOW_API_URL = "https://api.siliconflow.cn/v1/rerank" # 需根据实际文档确认
|
33 |
+
|
34 |
+
# 自定义 APIEmbeddings 类(使用 Hugging Face API 调用 BAAI/bge-m3)
|
35 |
+
class APIEmbeddings(Embeddings):
|
36 |
+
def __init__(self, model_name="BAAI/bge-m3"):
|
37 |
+
self.model_name = model_name
|
38 |
+
self.api_key = os.getenv("HUGGINGFACE_API_KEY")
|
39 |
+
if not self.api_key:
|
40 |
+
raise ValueError("HUGGINGFACE_API_KEY 未设置,请在环境变量中配置或在 .env 文件中添加")
|
41 |
+
|
42 |
+
def embed_documents(self, texts):
|
43 |
+
embeddings = []
|
44 |
+
batch_size = 1000 # 根据需要调整批次大小
|
45 |
+
|
46 |
+
for i in tqdm(range(0, len(texts), batch_size), desc="生成嵌入进度"):
|
47 |
+
batch_texts = texts[i:i + batch_size]
|
48 |
+
batch_embeddings = self._request_embeddings(batch_texts)
|
49 |
+
embeddings.extend(batch_embeddings)
|
50 |
+
|
51 |
+
return embeddings
|
52 |
+
|
53 |
+
def embed_query(self, text):
|
54 |
+
query_embeddings = self._request_embeddings([text])
|
55 |
+
return query_embeddings[0]
|
56 |
+
|
57 |
+
def _request_embeddings(self, texts):
|
58 |
+
headers = {
|
59 |
+
"Authorization": f"Bearer {self.api_key}",
|
60 |
+
"Content-Type": "application/json"
|
61 |
+
}
|
62 |
+
payload = {
|
63 |
+
"inputs": texts,
|
64 |
+
"model": self.model_name
|
65 |
+
}
|
66 |
+
|
67 |
+
response = requests.post("https://api-inference.huggingface.co/models/BAAI/bge-m3", headers=headers, json=payload)
|
68 |
+
response.raise_for_status()
|
69 |
+
|
70 |
+
return response.json()[0]["embedding"]
|
71 |
+
|
72 |
+
# 重排序函数,使用 SiliconFlow API 调用 BAAI/bge-reranker-v2-m3
|
73 |
+
def rerank_documents(query, documents, top_n=15):
|
74 |
+
try:
|
75 |
+
if not documents or not query:
|
76 |
+
raise ValueError("查询或文档列表为空")
|
77 |
+
|
78 |
+
# 提取文档内容和元数据,限制长度为 2048 字符
|
79 |
+
doc_texts = [(doc.page_content[:2048].replace("\n", " ").strip(), doc.metadata.get("book", "未知来源")) for doc in documents[:50]]
|
80 |
+
print(f"Query: {query[:100]}... (长度: {len(query)})")
|
81 |
+
print(f"文档数量 (前50个): {len(doc_texts)}")
|
82 |
+
for i, (doc, book) in enumerate(doc_texts[:5]): # 仅打印前5个用于调试
|
83 |
+
print(f" Doc {i}: {doc[:100]}... (来源: {book})")
|
84 |
+
|
85 |
+
# 构造 SiliconFlow API 请求
|
86 |
+
headers = {
|
87 |
+
"Authorization": f"Bearer {SILICONFLOW_API_KEY}",
|
88 |
+
"Content-Type": "application/json"
|
89 |
+
}
|
90 |
+
payload = {
|
91 |
+
"model": "BAAI/bge-reranker-v2-m3",
|
92 |
+
"query": query,
|
93 |
+
"documents": [text for text, _ in doc_texts],
|
94 |
+
"top_n": top_n
|
95 |
+
}
|
96 |
+
|
97 |
+
start_time = time.time()
|
98 |
+
response = requests.post(SILICONFLOW_API_URL, headers=headers, json=payload)
|
99 |
+
response.raise_for_status() # 检查请求是否成功
|
100 |
+
rerank_time = time.time() - start_time
|
101 |
+
print(f"重排序耗时: {rerank_time:.2f} 秒")
|
102 |
+
|
103 |
+
# 解析 SiliconFlow API 响应
|
104 |
+
result = response.json()
|
105 |
+
print(f"SiliconFlow API 响应: {result}")
|
106 |
+
|
107 |
+
# 验证返回结果
|
108 |
+
if "results" not in result or not isinstance(result["results"], list):
|
109 |
+
raise ValueError(f"SiliconFlow API 返回格式错误: {result}")
|
110 |
+
|
111 |
+
# 构建重排序结果,修正键名为 "relevance_score"
|
112 |
+
reranked_docs = []
|
113 |
+
for res in result["results"]:
|
114 |
+
if "index" not in res or "relevance_score" not in res:
|
115 |
+
raise ValueError(f"SiliconFlow API 返回的条目格式错误: {res}")
|
116 |
+
index = res["index"]
|
117 |
+
score = res["relevance_score"]
|
118 |
+
if index < len(documents):
|
119 |
+
text, book = doc_texts[index]
|
120 |
+
reranked_docs.append((Document(page_content=text, metadata={"book": book}), score))
|
121 |
+
|
122 |
+
# 按得分排序并截取 top_n
|
123 |
+
reranked_docs = sorted(reranked_docs, key=lambda x: x[1], reverse=True)[:top_n]
|
124 |
+
print(f"重排序结果 (数量: {len(reranked_docs)}):")
|
125 |
+
for i, (doc, score) in enumerate(reranked_docs):
|
126 |
+
print(f" Doc {i}: {doc.page_content[:100]}... (来源: {doc.metadata.get('book', '未知来源')}, 得分: {score:.4f})")
|
127 |
+
|
128 |
+
return reranked_docs
|
129 |
+
except Exception as e:
|
130 |
+
error_msg = str(e)
|
131 |
+
print(f"错误详情: {error_msg}")
|
132 |
+
raise Exception(f"重排序失败: {error_msg}")
|
133 |
+
|
134 |
+
# 构建 HNSW 索引
|
135 |
+
def build_hnsw_index(knowledge_base_path, index_path):
|
136 |
+
print("开始加载文档...")
|
137 |
+
start_time = time.time()
|
138 |
+
loader = DirectoryLoader(knowledge_base_path, glob="*.txt", loader_cls=lambda path: TextLoader(path, encoding="utf-8"), use_multithreading=False)
|
139 |
+
documents = loader.load()
|
140 |
+
load_time = time.time() - start_time
|
141 |
+
print(f"加载完成,共 {len(documents)} 个文档,耗时 {load_time:.2f} 秒")
|
142 |
+
|
143 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
144 |
+
if not os.path.exists("chunks.pkl"):
|
145 |
+
print("开始分片...")
|
146 |
+
start_time = time.time()
|
147 |
+
texts = []
|
148 |
+
total_chars = 0
|
149 |
+
total_bytes = 0
|
150 |
+
for i, doc in enumerate(documents):
|
151 |
+
doc_chunks = text_splitter.split_documents([doc])
|
152 |
+
for chunk in doc_chunks:
|
153 |
+
content = chunk.page_content
|
154 |
+
file_path = chunk.metadata.get("source", "")
|
155 |
+
book_name = os.path.basename(file_path).replace(".txt", "").replace("_", "·")
|
156 |
+
texts.append(Document(page_content=content, metadata={"book": book_name or "未知来源"}))
|
157 |
+
total_chars += len(content)
|
158 |
+
total_bytes += len(content.encode('utf-8'))
|
159 |
+
if i < 5:
|
160 |
+
print(f"文件 {i} 字符数: {len(doc.page_content)}, 字节数: {len(doc.page_content.encode('utf-8'))}, 来源: {file_path}")
|
161 |
+
if (i + 1) % 10 == 0:
|
162 |
+
print(f"分片进度: 已处理 {i + 1}/{len(documents)} 个文件,当前分片总数: {len(texts)}")
|
163 |
+
with open("chunks.pkl", "wb") as f:
|
164 |
+
pickle.dump(texts, f)
|
165 |
+
split_time = time.time() - start_time
|
166 |
+
print(f"分片完成,共 {len(texts)} 个 chunk,总字符数: {total_chars},总字节数: {total_bytes},耗时 {split_time:.2f} 秒")
|
167 |
+
else:
|
168 |
+
with open("chunks.pkl", "rb") as f:
|
169 |
+
texts = pickle.load(f)
|
170 |
+
print(f"加载已有分片,共 {len(texts)} 个 chunk")
|
171 |
+
|
172 |
+
if not os.path.exists("embeddings.npy"):
|
173 |
+
print("开始生成嵌入(使用 BAAI/bge-m3 API,分批处理)...")
|
174 |
+
embeddings = APIEmbeddings()
|
175 |
+
texts_content = [text.page_content for text in texts]
|
176 |
+
embeddings_array = embeddings.embed_documents(texts_content)
|
177 |
+
if os.path.exists("embeddings_temp.npy"):
|
178 |
+
os.remove("embeddings_temp.npy")
|
179 |
+
print(f"嵌入生成完成,维度: {embeddings_array.shape}")
|
180 |
+
else:
|
181 |
+
embeddings_array = np.load("embeddings.npy")
|
182 |
+
print(f"加载已有嵌入,维度: {embeddings_array.shape}")
|
183 |
+
|
184 |
+
dimension = embeddings_array.shape[1]
|
185 |
+
index = faiss.IndexHNSWFlat(dimension, 16)
|
186 |
+
index.hnsw.efConstruction = 100
|
187 |
+
print("开始构建 HNSW 索引...")
|
188 |
+
|
189 |
+
batch_size = 5000
|
190 |
+
total_vectors = embeddings_array.shape[0]
|
191 |
+
for i in range(0, total_vectors, batch_size):
|
192 |
+
batch = embeddings_array[i:i + batch_size]
|
193 |
+
index.add(batch)
|
194 |
+
print(f"索引构建进度: {min(i + batch_size, total_vectors)} / {total_vectors}")
|
195 |
+
|
196 |
+
text_embeddings = [(text.page_content, embeddings_array[i]) for i, text in enumerate(texts)]
|
197 |
+
vector_store = FAISS.from_embeddings(text_embeddings, embeddings, normalize_L2=True)
|
198 |
+
vector_store.index = index
|
199 |
+
vector_store.docstore._dict.clear()
|
200 |
+
vector_store.index_to_docstore_id.clear()
|
201 |
+
|
202 |
+
for i, text in enumerate(texts):
|
203 |
+
doc_id = str(i)
|
204 |
+
vector_store.docstore._dict[doc_id] = text
|
205 |
+
vector_store.index_to_docstore_id[i] = doc_id
|
206 |
+
|
207 |
+
print("开始保存索引...")
|
208 |
+
vector_store.save_local(index_path)
|
209 |
+
print(f"HNSW 索引已生成并保存到 '{index_path}'")
|
210 |
+
return vector_store, texts
|
211 |
+
|
212 |
+
# 初始化嵌入模型
|
213 |
+
embeddings = APIEmbeddings(model_name="BAAI/bge-m3")
|
214 |
+
print("已初始化 BAAI/bge-m3 嵌入模型,通过 API 调用")
|
215 |
+
|
216 |
+
# 加载或生成索引
|
217 |
+
index_path = "faiss_index_hnsw_new"
|
218 |
+
knowledge_base_path = "knowledge_base"
|
219 |
+
|
220 |
+
if not os.path.exists(index_path):
|
221 |
+
if os.path.exists(knowledge_base_path):
|
222 |
+
print("检测到 knowledge_base,正在生成 HNSW 索引...")
|
223 |
+
vector_store, all_documents = build_hnsw_index(knowledge_base_path, index_path)
|
224 |
+
else:
|
225 |
+
raise FileNotFoundError("未找到 'knowledge_base',请提供知识库数据")
|
226 |
+
else:
|
227 |
+
vector_store = FAISS.load_local(index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
|
228 |
+
vector_store.index.hnsw.efSearch = 300
|
229 |
+
print("已加载 HNSW 索引 'faiss_index_hnsw_new',efSearch 设置为 300")
|
230 |
+
with open("chunks.pkl", "rb") as f:
|
231 |
+
all_documents = pickle.load(f)
|
232 |
+
book_counts = {}
|
233 |
+
for doc in all_documents:
|
234 |
+
book = doc.metadata.get("book", "未知来源")
|
235 |
+
book_counts[book] = book_counts.get(book, 0) + 1
|
236 |
+
print(f"all_documents 书籍分布: {book_counts}")
|
237 |
+
|
238 |
+
# 初始化 ChatOpenAI
|
239 |
+
llm = ChatOpenAI(
|
240 |
+
model="deepseek/deepseek-r1:free",
|
241 |
+
api_key=os.environ["OPENROUTER_API_KEY"],
|
242 |
+
base_url="https://openrouter.ai/api/v1",
|
243 |
+
timeout=60,
|
244 |
+
temperature=0.3,
|
245 |
+
max_tokens=130000,
|
246 |
+
streaming=True
|
247 |
+
)
|
248 |
+
|
249 |
+
# 定义提示词模板
|
250 |
+
prompt_template = PromptTemplate(
|
251 |
+
input_variables=["context", "question", "chat_history"],
|
252 |
+
template="""
|
253 |
+
你是一个研究李敖的专家,根据用户提出的问题{question}、最近10轮对话历史{chat_history}以及从李敖相关书籍和评论中检索的至少10篇文本内容{context}回答问题。
|
254 |
+
在回答时,请注意以下几点:
|
255 |
+
- 结合李敖的写作风格和思想,筛选出与问题和对话历史最相关的检索内容,避免无关信息。
|
256 |
+
- 必须在回答中引用至少10篇不同的文本内容,引用格式为[引用: 文本序号],例如[引用: 1][引用: 2],并确保每篇文本在回答中都有明确使用。
|
257 |
+
- 在回答的末尾,必须以“引用文献”标题列出所有引用的文本序号及其内容摘要(每篇不超过50字)以及具体的书目信息(例如书名和章节),格式为:
|
258 |
+
- 引用文献:
|
259 |
+
1. [文本 1] 摘要... 出自:书名,第X页/章节。
|
260 |
+
2. [文本 2] 摘要... 出自:书名,第X页/章节。
|
261 |
+
(依此类推,至少10篇)
|
262 |
+
- 如果问题涉及李敖对某人或某事的评价,优先引用李敖的直接言论或文字,并说明出处。
|
263 |
+
- 回答应结构化、分段落,确保逻辑清晰,语言生动,类似李敖的犀利风格。
|
264 |
+
- 如果检索内容和历史不足以直接回答问题,可根据李敖的性格和观点推测其可能的看法,但需说明这是推测。
|
265 |
+
- 只能基于提供的知识库内容{context}和对话历史{chat_history}回答,不得引入外部信息。
|
266 |
+
- 对于列举类问题,控制在10个要点以内,并优先提供最相关项。
|
267 |
+
- 如果回答较长,结构化分段总结,分点作答控制在8个点以内。
|
268 |
+
- 对于客观类的问答,如果问题的答案非常简短,可以适当补充一到两句相关信息,以丰富内容。
|
269 |
+
- 你需要根据用户要求和回答内容选择合适、美观的回答格式,确保可读性强。
|
270 |
+
- 你的回答应该综合多个相关知识库内容来回答,不能重复引用一个知识库内容。
|
271 |
+
- 除非用户要求,否则你回答的语言需要和用户提问的语言保持一致。
|
272 |
+
"""
|
273 |
+
)
|
274 |
+
|
275 |
+
# 对话历史管理类
|
276 |
+
class ConversationHistory:
|
277 |
+
def __init__(self, max_length=10):
|
278 |
+
self.history = deque(maxlen=max_length)
|
279 |
+
|
280 |
+
def add_turn(self, question, answer):
|
281 |
+
self.history.append((question, answer))
|
282 |
+
|
283 |
+
def get_history(self):
|
284 |
+
return [(turn[0], turn[1]) for turn in self.history]
|
285 |
+
|
286 |
+
def clear(self):
|
287 |
+
self.history.clear()
|
288 |
+
|
289 |
+
# 用户会话状态类
|
290 |
+
class UserSession:
|
291 |
+
def __init__(self):
|
292 |
+
self.conversation = ConversationHistory()
|
293 |
+
self.output_queue = queue.Queue()
|
294 |
+
self.stop_flag = threading.Event()
|
295 |
+
|
296 |
+
# 生成回答的线程函数
|
297 |
+
def generate_answer_thread(question, session):
|
298 |
+
stop_flag = session.stop_flag
|
299 |
+
output_queue = session.output_queue
|
300 |
+
conversation = session.conversation
|
301 |
+
|
302 |
+
stop_flag.clear()
|
303 |
+
try:
|
304 |
+
history_list = conversation.get_history()
|
305 |
+
history_text = "\n".join([f"问: {q}\n答: {a}" for q, a in history_list]) if history_list else ""
|
306 |
+
query_with_context = f"{history_text}\n当前问题: {question}" if history_text else question
|
307 |
+
|
308 |
+
# 1. 使用 BAAI/bge-m3 API 生成查询嵌入
|
309 |
+
start_time = time.time()
|
310 |
+
embeddings = APIEmbeddings()
|
311 |
+
query_embedding = embeddings.embed_query(query_with_context)
|
312 |
+
embed_time = time.time() - start_time
|
313 |
+
output_queue.put(f"嵌入耗时 (BAAI/bge-m3 API): {embed_time:.2f} 秒\n")
|
314 |
+
|
315 |
+
if stop_flag.is_set():
|
316 |
+
output_queue.put("生成已停止")
|
317 |
+
return
|
318 |
+
|
319 |
+
# 2. 使用 FAISS HNSW 索引进行初始检索
|
320 |
+
start_time = time.time()
|
321 |
+
initial_docs_with_scores = vector_store.similarity_search_with_score(query_with_context, k=50)
|
322 |
+
search_time = time.time() - start_time
|
323 |
+
output_queue.put(f"初始检索数量: {len(initial_docs_with_scores)}\n检索耗时: {search_time:.2f} 秒\n")
|
324 |
+
|
325 |
+
if stop_flag.is_set():
|
326 |
+
output_queue.put("生成已停止")
|
327 |
+
return
|
328 |
+
|
329 |
+
initial_docs = [doc for doc, _ in initial_docs_with_scores]
|
330 |
+
|
331 |
+
# 3. 使用 SiliconFlow 的 BAAI/bge-reranker-v2-m3 进行重排序
|
332 |
+
start_time = time.time()
|
333 |
+
reranked_docs_with_scores = rerank_documents(query_with_context, initial_docs, top_n=15)
|
334 |
+
rerank_time = time.time() - start_time
|
335 |
+
output_queue.put(f"重排序耗时 (BAAI/bge-reranker-v2-m3): {rerank_time:.2f} 秒\n")
|
336 |
+
|
337 |
+
if stop_flag.is_set():
|
338 |
+
output_queue.put("生成已停止")
|
339 |
+
return
|
340 |
+
|
341 |
+
# 调整 final_docs 数量,取前 10 篇
|
342 |
+
final_docs = [doc for doc, _ in reranked_docs_with_scores][:10]
|
343 |
+
if len(final_docs) < 10:
|
344 |
+
output_queue.put(f"警告:仅检索到 {len(final_docs)} 篇文本,可能无法满足引用 10 篇的要求")
|
345 |
+
|
346 |
+
# 构造 context,包含文本内容和书目信息
|
347 |
+
context = "\n\n".join([f"[文本 {i+1}] {doc.page_content} (出处: {doc.metadata.get('book', '未知来源')})" for i, doc in enumerate(final_docs)])
|
348 |
+
chat_history = [HumanMessage(content=q) if i % 2 == 0 else AIMessage(content=a)
|
349 |
+
for i, (q, a) in enumerate(history_list)]
|
350 |
+
prompt = prompt_template.format(context=context, question=question, chat_history=history_text)
|
351 |
+
|
352 |
+
# 4. 使用 LLM 生成回答
|
353 |
+
answer = ""
|
354 |
+
start_time = time.time()
|
355 |
+
for chunk in llm.stream([HumanMessage(content=prompt)]):
|
356 |
+
if stop_flag.is_set():
|
357 |
+
output_queue.put(answer + "\n\n(生成已停止)")
|
358 |
+
return
|
359 |
+
answer += chunk.content
|
360 |
+
output_queue.put(answer)
|
361 |
+
llm_time = time.time() - start_time
|
362 |
+
output_queue.put(f"\nLLM 生成耗时: {llm_time:.2f} 秒")
|
363 |
+
|
364 |
+
conversation.add_turn(question, answer)
|
365 |
+
output_queue.put(answer)
|
366 |
+
|
367 |
+
except Exception as e:
|
368 |
+
output_queue.put(f"Error: {str(e)}")
|
369 |
+
|
370 |
+
# Gradio 接口函数
|
371 |
+
def answer_question(question, session_state):
|
372 |
+
if session_state is None:
|
373 |
+
session_state = UserSession()
|
374 |
+
|
375 |
+
thread = threading.Thread(target=generate_answer_thread, args=(question, session_state))
|
376 |
+
thread.start()
|
377 |
+
|
378 |
+
while thread.is_alive() or not session_state.output_queue.empty():
|
379 |
+
try:
|
380 |
+
output = session_state.output_queue.get(timeout=0.1)
|
381 |
+
yield output, session_state
|
382 |
+
except queue.Empty:
|
383 |
+
continue
|
384 |
+
|
385 |
+
while not session_state.output_queue.empty():
|
386 |
+
yield session_state.output_queue.get(), session_state
|
387 |
+
|
388 |
+
def stop_generation(session_state):
|
389 |
+
if session_state is not None:
|
390 |
+
session_state.stop_flag.set()
|
391 |
+
return "生成已停止,正在中止..."
|
392 |
+
|
393 |
+
def clear_conversation():
|
394 |
+
return "对话历史已清空,请开始新的对话。", UserSession()
|
395 |
+
|
396 |
+
# 创建 Gradio 界面
|
397 |
+
with gr.Blocks(title="AI李敖助手") as interface:
|
398 |
+
gr.Markdown("### AI李敖助手")
|
399 |
+
gr.Markdown("基于李敖163本相关书籍构建的知识库,支持上下文关联,记住最近10轮对话,输入问题以获取李敖风格的回答。")
|
400 |
+
|
401 |
+
session_state = gr.State(value=None)
|
402 |
+
|
403 |
+
with gr.Row():
|
404 |
+
with gr.Column(scale=3):
|
405 |
+
question_input = gr.Textbox(label="请输入您的问题", placeholder="输入您的问题...")
|
406 |
+
submit_button = gr.Button("提交")
|
407 |
+
with gr.Column(scale=1):
|
408 |
+
clear_button = gr.Button("新建对话")
|
409 |
+
stop_button = gr.Button("停止生成")
|
410 |
+
|
411 |
+
output_text = gr.Textbox(label="回答", interactive=False)
|
412 |
+
|
413 |
+
submit_button.click(fn=answer_question, inputs=[question_input, session_state], outputs=[output_text, session_state])
|
414 |
+
clear_button.click(fn=clear_conversation, inputs=None, outputs=[output_text, session_state])
|
415 |
+
stop_button.click(fn=stop_generation, inputs=[session_state], outputs=output_text)
|
416 |
+
|
417 |
+
# 启动应用
|
418 |
+
if __name__ == "__main__":
|
419 |
+
interface.launch(share=True)
|
faiss_index_hnsw_new/index.faiss
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e4149ca48ac3eb082e529c318ecec16fa8e512a38976efbfdf2d1a8980be4c69
|
3 |
+
size 193971462
|
faiss_index_hnsw_new/index.pkl
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2fd2179d6ec9a9537de8812fbe217136841905214c29d91cef8a9c76a96b2a1a
|
3 |
+
size 111802375
|
requirements.txt
ADDED
Binary file (3.92 kB). View file
|
|