Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files- .gitattributes +36 -35
- README.md +8 -14
- app.py +419 -0
- faiss_index_hnsw_new/index.faiss +3 -0
- faiss_index_hnsw_new/index.pkl +3 -0
- requirements.txt +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,36 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
faiss_index_hnsw_new/index.faiss filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,14 +1,8 @@
|
|
| 1 |
-
---
|
| 2 |
-
title:
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
pinned: false
|
| 10 |
-
license: mit
|
| 11 |
-
short_description: ai李敖测试服
|
| 12 |
-
---
|
| 13 |
-
|
| 14 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: aileeao_test
|
| 3 |
+
app_file: app.py
|
| 4 |
+
sdk: gradio
|
| 5 |
+
sdk_version: 5.20.0
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.py
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import gradio as gr
|
| 3 |
+
import requests
|
| 4 |
+
from langchain_community.document_loaders import TextLoader, DirectoryLoader
|
| 5 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 6 |
+
from langchain_community.vectorstores import FAISS
|
| 7 |
+
from langchain_openai import ChatOpenAI
|
| 8 |
+
from langchain.prompts import PromptTemplate
|
| 9 |
+
import numpy as np
|
| 10 |
+
import faiss
|
| 11 |
+
from collections import deque
|
| 12 |
+
from langchain_core.embeddings import Embeddings
|
| 13 |
+
import threading
|
| 14 |
+
import queue
|
| 15 |
+
from langchain_core.messages import HumanMessage, AIMessage
|
| 16 |
+
from sentence_transformers import SentenceTransformer
|
| 17 |
+
import pickle
|
| 18 |
+
import torch
|
| 19 |
+
from langchain_core.documents import Document
|
| 20 |
+
import time
|
| 21 |
+
from tqdm import tqdm
|
| 22 |
+
|
| 23 |
+
# 获取环境变量
|
| 24 |
+
os.environ["OPENROUTER_API_KEY"] = os.getenv("OPENROUTER_API_KEY", "")
|
| 25 |
+
if not os.environ["OPENROUTER_API_KEY"]:
|
| 26 |
+
raise ValueError("OPENROUTER_API_KEY 未设置,请在环境变量中配置或在 .env 文件中添加")
|
| 27 |
+
SILICONFLOW_API_KEY = os.getenv("SILICONFLOW_API_KEY")
|
| 28 |
+
if not SILICONFLOW_API_KEY:
|
| 29 |
+
raise ValueError("SILICONFLOW_API_KEY 未设置,请在 Hugging Face Spaces 的 Settings > Secrets 中添加 SILICONFLOW_API_KEY")
|
| 30 |
+
|
| 31 |
+
# SiliconFlow API 配置
|
| 32 |
+
SILICONFLOW_API_URL = "https://api.siliconflow.cn/v1/rerank" # 需根据实际文档确认
|
| 33 |
+
|
| 34 |
+
# 自定义 APIEmbeddings 类(使用 Hugging Face API 调用 BAAI/bge-m3)
|
| 35 |
+
class APIEmbeddings(Embeddings):
|
| 36 |
+
def __init__(self, model_name="BAAI/bge-m3"):
|
| 37 |
+
self.model_name = model_name
|
| 38 |
+
self.api_key = os.getenv("HUGGINGFACE_API_KEY")
|
| 39 |
+
if not self.api_key:
|
| 40 |
+
raise ValueError("HUGGINGFACE_API_KEY 未设置,请在环境变量中配置或在 .env 文件中添加")
|
| 41 |
+
|
| 42 |
+
def embed_documents(self, texts):
|
| 43 |
+
embeddings = []
|
| 44 |
+
batch_size = 1000 # 根据需要调整批次大小
|
| 45 |
+
|
| 46 |
+
for i in tqdm(range(0, len(texts), batch_size), desc="生成嵌入进度"):
|
| 47 |
+
batch_texts = texts[i:i + batch_size]
|
| 48 |
+
batch_embeddings = self._request_embeddings(batch_texts)
|
| 49 |
+
embeddings.extend(batch_embeddings)
|
| 50 |
+
|
| 51 |
+
return embeddings
|
| 52 |
+
|
| 53 |
+
def embed_query(self, text):
|
| 54 |
+
query_embeddings = self._request_embeddings([text])
|
| 55 |
+
return query_embeddings[0]
|
| 56 |
+
|
| 57 |
+
def _request_embeddings(self, texts):
|
| 58 |
+
headers = {
|
| 59 |
+
"Authorization": f"Bearer {self.api_key}",
|
| 60 |
+
"Content-Type": "application/json"
|
| 61 |
+
}
|
| 62 |
+
payload = {
|
| 63 |
+
"inputs": texts,
|
| 64 |
+
"model": self.model_name
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
response = requests.post("https://api-inference.huggingface.co/models/BAAI/bge-m3", headers=headers, json=payload)
|
| 68 |
+
response.raise_for_status()
|
| 69 |
+
|
| 70 |
+
return response.json()[0]["embedding"]
|
| 71 |
+
|
| 72 |
+
# 重排序函数,使用 SiliconFlow API 调用 BAAI/bge-reranker-v2-m3
|
| 73 |
+
def rerank_documents(query, documents, top_n=15):
|
| 74 |
+
try:
|
| 75 |
+
if not documents or not query:
|
| 76 |
+
raise ValueError("查询或文档列表为空")
|
| 77 |
+
|
| 78 |
+
# 提取文档内容和元数据,限制长度为 2048 字符
|
| 79 |
+
doc_texts = [(doc.page_content[:2048].replace("\n", " ").strip(), doc.metadata.get("book", "未知来源")) for doc in documents[:50]]
|
| 80 |
+
print(f"Query: {query[:100]}... (长度: {len(query)})")
|
| 81 |
+
print(f"文档数量 (前50个): {len(doc_texts)}")
|
| 82 |
+
for i, (doc, book) in enumerate(doc_texts[:5]): # 仅打印前5个用于调试
|
| 83 |
+
print(f" Doc {i}: {doc[:100]}... (来源: {book})")
|
| 84 |
+
|
| 85 |
+
# 构造 SiliconFlow API 请求
|
| 86 |
+
headers = {
|
| 87 |
+
"Authorization": f"Bearer {SILICONFLOW_API_KEY}",
|
| 88 |
+
"Content-Type": "application/json"
|
| 89 |
+
}
|
| 90 |
+
payload = {
|
| 91 |
+
"model": "BAAI/bge-reranker-v2-m3",
|
| 92 |
+
"query": query,
|
| 93 |
+
"documents": [text for text, _ in doc_texts],
|
| 94 |
+
"top_n": top_n
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
start_time = time.time()
|
| 98 |
+
response = requests.post(SILICONFLOW_API_URL, headers=headers, json=payload)
|
| 99 |
+
response.raise_for_status() # 检查请求是否成功
|
| 100 |
+
rerank_time = time.time() - start_time
|
| 101 |
+
print(f"重排序耗时: {rerank_time:.2f} 秒")
|
| 102 |
+
|
| 103 |
+
# 解析 SiliconFlow API 响应
|
| 104 |
+
result = response.json()
|
| 105 |
+
print(f"SiliconFlow API 响应: {result}")
|
| 106 |
+
|
| 107 |
+
# 验证返回结果
|
| 108 |
+
if "results" not in result or not isinstance(result["results"], list):
|
| 109 |
+
raise ValueError(f"SiliconFlow API 返回格式错误: {result}")
|
| 110 |
+
|
| 111 |
+
# 构建重排序结果,修正键名为 "relevance_score"
|
| 112 |
+
reranked_docs = []
|
| 113 |
+
for res in result["results"]:
|
| 114 |
+
if "index" not in res or "relevance_score" not in res:
|
| 115 |
+
raise ValueError(f"SiliconFlow API 返回的条目格式错误: {res}")
|
| 116 |
+
index = res["index"]
|
| 117 |
+
score = res["relevance_score"]
|
| 118 |
+
if index < len(documents):
|
| 119 |
+
text, book = doc_texts[index]
|
| 120 |
+
reranked_docs.append((Document(page_content=text, metadata={"book": book}), score))
|
| 121 |
+
|
| 122 |
+
# 按得分排序并截取 top_n
|
| 123 |
+
reranked_docs = sorted(reranked_docs, key=lambda x: x[1], reverse=True)[:top_n]
|
| 124 |
+
print(f"重排序结果 (数量: {len(reranked_docs)}):")
|
| 125 |
+
for i, (doc, score) in enumerate(reranked_docs):
|
| 126 |
+
print(f" Doc {i}: {doc.page_content[:100]}... (来源: {doc.metadata.get('book', '未知来源')}, 得分: {score:.4f})")
|
| 127 |
+
|
| 128 |
+
return reranked_docs
|
| 129 |
+
except Exception as e:
|
| 130 |
+
error_msg = str(e)
|
| 131 |
+
print(f"错误详情: {error_msg}")
|
| 132 |
+
raise Exception(f"重排序失败: {error_msg}")
|
| 133 |
+
|
| 134 |
+
# 构建 HNSW 索引
|
| 135 |
+
def build_hnsw_index(knowledge_base_path, index_path):
|
| 136 |
+
print("开始加载文档...")
|
| 137 |
+
start_time = time.time()
|
| 138 |
+
loader = DirectoryLoader(knowledge_base_path, glob="*.txt", loader_cls=lambda path: TextLoader(path, encoding="utf-8"), use_multithreading=False)
|
| 139 |
+
documents = loader.load()
|
| 140 |
+
load_time = time.time() - start_time
|
| 141 |
+
print(f"加载完成,共 {len(documents)} 个文档,耗时 {load_time:.2f} 秒")
|
| 142 |
+
|
| 143 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
| 144 |
+
if not os.path.exists("chunks.pkl"):
|
| 145 |
+
print("开始分片...")
|
| 146 |
+
start_time = time.time()
|
| 147 |
+
texts = []
|
| 148 |
+
total_chars = 0
|
| 149 |
+
total_bytes = 0
|
| 150 |
+
for i, doc in enumerate(documents):
|
| 151 |
+
doc_chunks = text_splitter.split_documents([doc])
|
| 152 |
+
for chunk in doc_chunks:
|
| 153 |
+
content = chunk.page_content
|
| 154 |
+
file_path = chunk.metadata.get("source", "")
|
| 155 |
+
book_name = os.path.basename(file_path).replace(".txt", "").replace("_", "·")
|
| 156 |
+
texts.append(Document(page_content=content, metadata={"book": book_name or "未知来源"}))
|
| 157 |
+
total_chars += len(content)
|
| 158 |
+
total_bytes += len(content.encode('utf-8'))
|
| 159 |
+
if i < 5:
|
| 160 |
+
print(f"文件 {i} 字符数: {len(doc.page_content)}, 字节数: {len(doc.page_content.encode('utf-8'))}, 来源: {file_path}")
|
| 161 |
+
if (i + 1) % 10 == 0:
|
| 162 |
+
print(f"分片进度: 已处理 {i + 1}/{len(documents)} 个文件,当前分片总数: {len(texts)}")
|
| 163 |
+
with open("chunks.pkl", "wb") as f:
|
| 164 |
+
pickle.dump(texts, f)
|
| 165 |
+
split_time = time.time() - start_time
|
| 166 |
+
print(f"分片完成,共 {len(texts)} 个 chunk,总字符数: {total_chars},总字节数: {total_bytes},耗时 {split_time:.2f} 秒")
|
| 167 |
+
else:
|
| 168 |
+
with open("chunks.pkl", "rb") as f:
|
| 169 |
+
texts = pickle.load(f)
|
| 170 |
+
print(f"加载已有分片,共 {len(texts)} 个 chunk")
|
| 171 |
+
|
| 172 |
+
if not os.path.exists("embeddings.npy"):
|
| 173 |
+
print("开始生成嵌入(使用 BAAI/bge-m3 API,分批处理)...")
|
| 174 |
+
embeddings = APIEmbeddings()
|
| 175 |
+
texts_content = [text.page_content for text in texts]
|
| 176 |
+
embeddings_array = embeddings.embed_documents(texts_content)
|
| 177 |
+
if os.path.exists("embeddings_temp.npy"):
|
| 178 |
+
os.remove("embeddings_temp.npy")
|
| 179 |
+
print(f"嵌入生成完成,维度: {embeddings_array.shape}")
|
| 180 |
+
else:
|
| 181 |
+
embeddings_array = np.load("embeddings.npy")
|
| 182 |
+
print(f"加载已有嵌入,维度: {embeddings_array.shape}")
|
| 183 |
+
|
| 184 |
+
dimension = embeddings_array.shape[1]
|
| 185 |
+
index = faiss.IndexHNSWFlat(dimension, 16)
|
| 186 |
+
index.hnsw.efConstruction = 100
|
| 187 |
+
print("开始构建 HNSW 索引...")
|
| 188 |
+
|
| 189 |
+
batch_size = 5000
|
| 190 |
+
total_vectors = embeddings_array.shape[0]
|
| 191 |
+
for i in range(0, total_vectors, batch_size):
|
| 192 |
+
batch = embeddings_array[i:i + batch_size]
|
| 193 |
+
index.add(batch)
|
| 194 |
+
print(f"索引构建进度: {min(i + batch_size, total_vectors)} / {total_vectors}")
|
| 195 |
+
|
| 196 |
+
text_embeddings = [(text.page_content, embeddings_array[i]) for i, text in enumerate(texts)]
|
| 197 |
+
vector_store = FAISS.from_embeddings(text_embeddings, embeddings, normalize_L2=True)
|
| 198 |
+
vector_store.index = index
|
| 199 |
+
vector_store.docstore._dict.clear()
|
| 200 |
+
vector_store.index_to_docstore_id.clear()
|
| 201 |
+
|
| 202 |
+
for i, text in enumerate(texts):
|
| 203 |
+
doc_id = str(i)
|
| 204 |
+
vector_store.docstore._dict[doc_id] = text
|
| 205 |
+
vector_store.index_to_docstore_id[i] = doc_id
|
| 206 |
+
|
| 207 |
+
print("开始保存索引...")
|
| 208 |
+
vector_store.save_local(index_path)
|
| 209 |
+
print(f"HNSW 索引已生成并保存到 '{index_path}'")
|
| 210 |
+
return vector_store, texts
|
| 211 |
+
|
| 212 |
+
# 初始化嵌入模型
|
| 213 |
+
embeddings = APIEmbeddings(model_name="BAAI/bge-m3")
|
| 214 |
+
print("已初始化 BAAI/bge-m3 嵌入模型,通过 API 调用")
|
| 215 |
+
|
| 216 |
+
# 加载或生成索引
|
| 217 |
+
index_path = "faiss_index_hnsw_new"
|
| 218 |
+
knowledge_base_path = "knowledge_base"
|
| 219 |
+
|
| 220 |
+
if not os.path.exists(index_path):
|
| 221 |
+
if os.path.exists(knowledge_base_path):
|
| 222 |
+
print("检测到 knowledge_base,正在生成 HNSW 索引...")
|
| 223 |
+
vector_store, all_documents = build_hnsw_index(knowledge_base_path, index_path)
|
| 224 |
+
else:
|
| 225 |
+
raise FileNotFoundError("未找到 'knowledge_base',请提供知识库数据")
|
| 226 |
+
else:
|
| 227 |
+
vector_store = FAISS.load_local(index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
|
| 228 |
+
vector_store.index.hnsw.efSearch = 300
|
| 229 |
+
print("已加载 HNSW 索引 'faiss_index_hnsw_new',efSearch 设置为 300")
|
| 230 |
+
with open("chunks.pkl", "rb") as f:
|
| 231 |
+
all_documents = pickle.load(f)
|
| 232 |
+
book_counts = {}
|
| 233 |
+
for doc in all_documents:
|
| 234 |
+
book = doc.metadata.get("book", "未知来源")
|
| 235 |
+
book_counts[book] = book_counts.get(book, 0) + 1
|
| 236 |
+
print(f"all_documents 书籍分布: {book_counts}")
|
| 237 |
+
|
| 238 |
+
# 初始化 ChatOpenAI
|
| 239 |
+
llm = ChatOpenAI(
|
| 240 |
+
model="deepseek/deepseek-r1:free",
|
| 241 |
+
api_key=os.environ["OPENROUTER_API_KEY"],
|
| 242 |
+
base_url="https://openrouter.ai/api/v1",
|
| 243 |
+
timeout=60,
|
| 244 |
+
temperature=0.3,
|
| 245 |
+
max_tokens=130000,
|
| 246 |
+
streaming=True
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
# 定义提示词模板
|
| 250 |
+
prompt_template = PromptTemplate(
|
| 251 |
+
input_variables=["context", "question", "chat_history"],
|
| 252 |
+
template="""
|
| 253 |
+
你是一个研究李敖的专家,根据用户提出的问题{question}、最近10轮对话历史{chat_history}以及从李敖相关书籍和评论中检索的至少10篇文本内容{context}回答问题。
|
| 254 |
+
在回答时,请注意以下几点:
|
| 255 |
+
- 结合李敖的写作风格和思想,筛选出与问题和对话历史最相关的检索内容,避免无关信息。
|
| 256 |
+
- 必须在回答中引用至少10篇不同的文本内容,引用格式为[引用: 文本序号],例如[引用: 1][引用: 2],并确保每篇文本在回答中都有明确使用。
|
| 257 |
+
- 在回答的末尾,必须以“引用文献”标题列出所有引用的文本序号及其内容摘要(每篇不超过50字)以及具体的书目信息(例如书名和章节),格式为:
|
| 258 |
+
- 引用文献:
|
| 259 |
+
1. [文本 1] 摘要... 出自:书名,第X页/章节。
|
| 260 |
+
2. [文本 2] 摘要... 出自:书名,第X页/章节。
|
| 261 |
+
(依此类推,至少10篇)
|
| 262 |
+
- 如果问题涉及李敖对某人或某事的评价,优先引用李敖的直接言论或文字,并说明出处。
|
| 263 |
+
- 回答应结构化、分段落,确保逻辑清晰,语言生动,类似李敖的犀利风格。
|
| 264 |
+
- 如果检索内容和历史不足以直接回答问题,可根据李敖的性格和观点推测其可能的看法,但需说明这是推测。
|
| 265 |
+
- 只能基于提供的知识库内容{context}和对话历史{chat_history}回答,不得引入外部信息。
|
| 266 |
+
- 对于列举类问题,控制在10个要点以内,并优先提供最相关项。
|
| 267 |
+
- 如果回答较长,结构化分段总结,分点作答控制在8个点以内。
|
| 268 |
+
- 对于客观类的问答,如果问题的答案非常简短,可以适当补充一到两句相关信息,以丰富内容。
|
| 269 |
+
- 你需要根据用户要求和回答内容选择合适、美观的回答格式,确保可读性强。
|
| 270 |
+
- 你的回答应该综合多个相关知识库内容来回答,不能重复引用一个知识库内容。
|
| 271 |
+
- 除非用户要求,否则你回答的语言需要和用户提问的语言保持一致。
|
| 272 |
+
"""
|
| 273 |
+
)
|
| 274 |
+
|
| 275 |
+
# 对话历史管理类
|
| 276 |
+
class ConversationHistory:
|
| 277 |
+
def __init__(self, max_length=10):
|
| 278 |
+
self.history = deque(maxlen=max_length)
|
| 279 |
+
|
| 280 |
+
def add_turn(self, question, answer):
|
| 281 |
+
self.history.append((question, answer))
|
| 282 |
+
|
| 283 |
+
def get_history(self):
|
| 284 |
+
return [(turn[0], turn[1]) for turn in self.history]
|
| 285 |
+
|
| 286 |
+
def clear(self):
|
| 287 |
+
self.history.clear()
|
| 288 |
+
|
| 289 |
+
# 用户会话状态类
|
| 290 |
+
class UserSession:
|
| 291 |
+
def __init__(self):
|
| 292 |
+
self.conversation = ConversationHistory()
|
| 293 |
+
self.output_queue = queue.Queue()
|
| 294 |
+
self.stop_flag = threading.Event()
|
| 295 |
+
|
| 296 |
+
# 生成回答的线程函数
|
| 297 |
+
def generate_answer_thread(question, session):
|
| 298 |
+
stop_flag = session.stop_flag
|
| 299 |
+
output_queue = session.output_queue
|
| 300 |
+
conversation = session.conversation
|
| 301 |
+
|
| 302 |
+
stop_flag.clear()
|
| 303 |
+
try:
|
| 304 |
+
history_list = conversation.get_history()
|
| 305 |
+
history_text = "\n".join([f"问: {q}\n答: {a}" for q, a in history_list]) if history_list else ""
|
| 306 |
+
query_with_context = f"{history_text}\n当前问题: {question}" if history_text else question
|
| 307 |
+
|
| 308 |
+
# 1. 使用 BAAI/bge-m3 API 生成查询嵌入
|
| 309 |
+
start_time = time.time()
|
| 310 |
+
embeddings = APIEmbeddings()
|
| 311 |
+
query_embedding = embeddings.embed_query(query_with_context)
|
| 312 |
+
embed_time = time.time() - start_time
|
| 313 |
+
output_queue.put(f"嵌入耗时 (BAAI/bge-m3 API): {embed_time:.2f} 秒\n")
|
| 314 |
+
|
| 315 |
+
if stop_flag.is_set():
|
| 316 |
+
output_queue.put("生成已停止")
|
| 317 |
+
return
|
| 318 |
+
|
| 319 |
+
# 2. 使用 FAISS HNSW 索引进行初始检索
|
| 320 |
+
start_time = time.time()
|
| 321 |
+
initial_docs_with_scores = vector_store.similarity_search_with_score(query_with_context, k=50)
|
| 322 |
+
search_time = time.time() - start_time
|
| 323 |
+
output_queue.put(f"初始检索数量: {len(initial_docs_with_scores)}\n检索耗时: {search_time:.2f} 秒\n")
|
| 324 |
+
|
| 325 |
+
if stop_flag.is_set():
|
| 326 |
+
output_queue.put("生成已停止")
|
| 327 |
+
return
|
| 328 |
+
|
| 329 |
+
initial_docs = [doc for doc, _ in initial_docs_with_scores]
|
| 330 |
+
|
| 331 |
+
# 3. 使用 SiliconFlow 的 BAAI/bge-reranker-v2-m3 进行重排序
|
| 332 |
+
start_time = time.time()
|
| 333 |
+
reranked_docs_with_scores = rerank_documents(query_with_context, initial_docs, top_n=15)
|
| 334 |
+
rerank_time = time.time() - start_time
|
| 335 |
+
output_queue.put(f"重排序耗时 (BAAI/bge-reranker-v2-m3): {rerank_time:.2f} 秒\n")
|
| 336 |
+
|
| 337 |
+
if stop_flag.is_set():
|
| 338 |
+
output_queue.put("生成已停止")
|
| 339 |
+
return
|
| 340 |
+
|
| 341 |
+
# 调整 final_docs 数量,取前 10 篇
|
| 342 |
+
final_docs = [doc for doc, _ in reranked_docs_with_scores][:10]
|
| 343 |
+
if len(final_docs) < 10:
|
| 344 |
+
output_queue.put(f"警告:仅检索到 {len(final_docs)} 篇文本,可能无法满足引用 10 篇的要求")
|
| 345 |
+
|
| 346 |
+
# 构造 context,包含文本内容和书目信息
|
| 347 |
+
context = "\n\n".join([f"[文本 {i+1}] {doc.page_content} (出处: {doc.metadata.get('book', '未知来源')})" for i, doc in enumerate(final_docs)])
|
| 348 |
+
chat_history = [HumanMessage(content=q) if i % 2 == 0 else AIMessage(content=a)
|
| 349 |
+
for i, (q, a) in enumerate(history_list)]
|
| 350 |
+
prompt = prompt_template.format(context=context, question=question, chat_history=history_text)
|
| 351 |
+
|
| 352 |
+
# 4. 使用 LLM 生成回答
|
| 353 |
+
answer = ""
|
| 354 |
+
start_time = time.time()
|
| 355 |
+
for chunk in llm.stream([HumanMessage(content=prompt)]):
|
| 356 |
+
if stop_flag.is_set():
|
| 357 |
+
output_queue.put(answer + "\n\n(生成已停止)")
|
| 358 |
+
return
|
| 359 |
+
answer += chunk.content
|
| 360 |
+
output_queue.put(answer)
|
| 361 |
+
llm_time = time.time() - start_time
|
| 362 |
+
output_queue.put(f"\nLLM 生成耗时: {llm_time:.2f} 秒")
|
| 363 |
+
|
| 364 |
+
conversation.add_turn(question, answer)
|
| 365 |
+
output_queue.put(answer)
|
| 366 |
+
|
| 367 |
+
except Exception as e:
|
| 368 |
+
output_queue.put(f"Error: {str(e)}")
|
| 369 |
+
|
| 370 |
+
# Gradio 接口函数
|
| 371 |
+
def answer_question(question, session_state):
|
| 372 |
+
if session_state is None:
|
| 373 |
+
session_state = UserSession()
|
| 374 |
+
|
| 375 |
+
thread = threading.Thread(target=generate_answer_thread, args=(question, session_state))
|
| 376 |
+
thread.start()
|
| 377 |
+
|
| 378 |
+
while thread.is_alive() or not session_state.output_queue.empty():
|
| 379 |
+
try:
|
| 380 |
+
output = session_state.output_queue.get(timeout=0.1)
|
| 381 |
+
yield output, session_state
|
| 382 |
+
except queue.Empty:
|
| 383 |
+
continue
|
| 384 |
+
|
| 385 |
+
while not session_state.output_queue.empty():
|
| 386 |
+
yield session_state.output_queue.get(), session_state
|
| 387 |
+
|
| 388 |
+
def stop_generation(session_state):
|
| 389 |
+
if session_state is not None:
|
| 390 |
+
session_state.stop_flag.set()
|
| 391 |
+
return "生成已停止,正在中止..."
|
| 392 |
+
|
| 393 |
+
def clear_conversation():
|
| 394 |
+
return "对话历史已清空,请开始新的对话。", UserSession()
|
| 395 |
+
|
| 396 |
+
# 创建 Gradio 界面
|
| 397 |
+
with gr.Blocks(title="AI李敖助手") as interface:
|
| 398 |
+
gr.Markdown("### AI李敖助手")
|
| 399 |
+
gr.Markdown("基于李敖163本相关书籍构建的知识库,支持上下文关联,记住最近10轮对话,输入问题以获取李敖风格的回答。")
|
| 400 |
+
|
| 401 |
+
session_state = gr.State(value=None)
|
| 402 |
+
|
| 403 |
+
with gr.Row():
|
| 404 |
+
with gr.Column(scale=3):
|
| 405 |
+
question_input = gr.Textbox(label="请输入您的问题", placeholder="输入您的问题...")
|
| 406 |
+
submit_button = gr.Button("提交")
|
| 407 |
+
with gr.Column(scale=1):
|
| 408 |
+
clear_button = gr.Button("新建对话")
|
| 409 |
+
stop_button = gr.Button("停止生成")
|
| 410 |
+
|
| 411 |
+
output_text = gr.Textbox(label="回答", interactive=False)
|
| 412 |
+
|
| 413 |
+
submit_button.click(fn=answer_question, inputs=[question_input, session_state], outputs=[output_text, session_state])
|
| 414 |
+
clear_button.click(fn=clear_conversation, inputs=None, outputs=[output_text, session_state])
|
| 415 |
+
stop_button.click(fn=stop_generation, inputs=[session_state], outputs=output_text)
|
| 416 |
+
|
| 417 |
+
# 启动应用
|
| 418 |
+
if __name__ == "__main__":
|
| 419 |
+
interface.launch(share=True)
|
faiss_index_hnsw_new/index.faiss
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e4149ca48ac3eb082e529c318ecec16fa8e512a38976efbfdf2d1a8980be4c69
|
| 3 |
+
size 193971462
|
faiss_index_hnsw_new/index.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2fd2179d6ec9a9537de8812fbe217136841905214c29d91cef8a9c76a96b2a1a
|
| 3 |
+
size 111802375
|
requirements.txt
ADDED
|
Binary file (3.92 kB). View file
|
|
|